From 064919348dc344682ef16d374ac56022a92f5932 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 12 Jun 2019 21:43:43 -0500
Subject: [PATCH 001/459] Split off bytecode compilation into a separate crate

---
 Cargo.toml         |   13 +
 src/bytecode.rs    |  446 +++++++++++
 src/compile.rs     | 1874 ++++++++++++++++++++++++++++++++++++++++++++
 src/error.rs       |   67 ++
 src/lib.rs         |    7 +
 src/symboltable.rs |  588 ++++++++++++++
 6 files changed, 2995 insertions(+)
 create mode 100644 Cargo.toml
 create mode 100644 src/bytecode.rs
 create mode 100644 src/compile.rs
 create mode 100644 src/error.rs
 create mode 100644 src/lib.rs
 create mode 100644 src/symboltable.rs

diff --git a/Cargo.toml b/Cargo.toml
new file mode 100644
index 00000000..fc0d1959
--- /dev/null
+++ b/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+name = "rustpython_compiler"
+version = "0.1.0"
+authors = ["coolreader18 <33094578+coolreader18@users.noreply.github.com>"]
+edition = "2018"
+
+[dependencies]
+bitflags = "1.1"
+rustpython_parser = { path = "../parser" }
+serde = { version = "1.0", features = ["derive"] }
+num-complex = { version = "0.2", features = ["serde"] }
+num-bigint = { version = "0.2", features = ["serde"] }
+log = "0.3"
diff --git a/src/bytecode.rs b/src/bytecode.rs
new file mode 100644
index 00000000..0fc226b5
--- /dev/null
+++ b/src/bytecode.rs
@@ -0,0 +1,446 @@
+//! Implement python as a virtual machine with bytecodes. This module
+//! implements bytecode structure.
+
+/*
+ * Primitive instruction type, which can be encoded and decoded.
+ */
+
+use bitflags::bitflags;
+use num_bigint::BigInt;
+use num_complex::Complex64;
+use rustpython_parser::ast;
+use serde::{Deserialize, Serialize};
+use std::collections::{HashMap, HashSet};
+use std::fmt;
+
+/// Primary container of a single code object. Each python function has
+/// a codeobject. Also a module has a codeobject.
+#[derive(Clone, PartialEq, Serialize, Deserialize)]
+pub struct CodeObject {
+    pub instructions: Vec<Instruction>,
+    /// Jump targets.
+    pub label_map: HashMap<Label, usize>,
+    pub locations: Vec<ast::Location>,
+    pub arg_names: Vec<String>, // Names of positional arguments
+    pub varargs: Varargs,       // *args or *
+    pub kwonlyarg_names: Vec<String>,
+    pub varkeywords: Varargs, // **kwargs or **
+    pub source_path: String,
+    pub first_line_number: usize,
+    pub obj_name: String, // Name of the object that created this code object
+    pub is_generator: bool,
+}
+
+bitflags! {
+    #[derive(Serialize, Deserialize)]
+    pub struct FunctionOpArg: u8 {
+        const HAS_DEFAULTS = 0x01;
+        const HAS_KW_ONLY_DEFAULTS = 0x02;
+        const HAS_ANNOTATIONS = 0x04;
+    }
+}
+
+pub type Label = usize;
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum NameScope {
+    Local,
+    NonLocal,
+    Global,
+}
+
+/// A Single bytecode instruction.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum Instruction {
+    Import {
+        name: String,
+        symbol: Option<String>,
+    },
+    ImportStar {
+        name: String,
+    },
+    LoadName {
+        name: String,
+        scope: NameScope,
+    },
+    StoreName {
+        name: String,
+        scope: NameScope,
+    },
+    DeleteName {
+        name: String,
+    },
+    StoreSubscript,
+    DeleteSubscript,
+    StoreAttr {
+        name: String,
+    },
+    DeleteAttr {
+        name: String,
+    },
+    LoadConst {
+        value: Constant,
+    },
+    UnaryOperation {
+        op: UnaryOperator,
+    },
+    BinaryOperation {
+        op: BinaryOperator,
+        inplace: bool,
+    },
+    LoadAttr {
+        name: String,
+    },
+    CompareOperation {
+        op: ComparisonOperator,
+    },
+    Pop,
+    Rotate {
+        amount: usize,
+    },
+    Duplicate,
+    GetIter,
+    Pass,
+    Continue,
+    Break,
+    Jump {
+        target: Label,
+    },
+    JumpIf {
+        target: Label,
+    },
+    JumpIfFalse {
+        target: Label,
+    },
+    MakeFunction {
+        flags: FunctionOpArg,
+    },
+    CallFunction {
+        typ: CallType,
+    },
+    ForIter {
+        target: Label,
+    },
+    ReturnValue,
+    YieldValue,
+    YieldFrom,
+    SetupLoop {
+        start: Label,
+        end: Label,
+    },
+    SetupExcept {
+        handler: Label,
+    },
+    SetupWith {
+        end: Label,
+    },
+    CleanupWith {
+        end: Label,
+    },
+    PopBlock,
+    Raise {
+        argc: usize,
+    },
+    BuildString {
+        size: usize,
+    },
+    BuildTuple {
+        size: usize,
+        unpack: bool,
+    },
+    BuildList {
+        size: usize,
+        unpack: bool,
+    },
+    BuildSet {
+        size: usize,
+        unpack: bool,
+    },
+    BuildMap {
+        size: usize,
+        unpack: bool,
+    },
+    BuildSlice {
+        size: usize,
+    },
+    ListAppend {
+        i: usize,
+    },
+    SetAdd {
+        i: usize,
+    },
+    MapAdd {
+        i: usize,
+    },
+    PrintExpr,
+    LoadBuildClass,
+    UnpackSequence {
+        size: usize,
+    },
+    UnpackEx {
+        before: usize,
+        after: usize,
+    },
+    Unpack,
+    FormatValue {
+        conversion: Option<ast::ConversionFlag>,
+        spec: String,
+    },
+    PopException,
+}
+
+use self::Instruction::*;
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum CallType {
+    Positional(usize),
+    Keyword(usize),
+    Ex(bool),
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum Constant {
+    Integer { value: BigInt },
+    Float { value: f64 },
+    Complex { value: Complex64 },
+    Boolean { value: bool },
+    String { value: String },
+    Bytes { value: Vec<u8> },
+    Code { code: Box<CodeObject> },
+    Tuple { elements: Vec<Constant> },
+    None,
+    Ellipsis,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum ComparisonOperator {
+    Greater,
+    GreaterOrEqual,
+    Less,
+    LessOrEqual,
+    Equal,
+    NotEqual,
+    In,
+    NotIn,
+    Is,
+    IsNot,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum BinaryOperator {
+    Power,
+    Multiply,
+    MatrixMultiply,
+    Divide,
+    FloorDivide,
+    Modulo,
+    Add,
+    Subtract,
+    Subscript,
+    Lshift,
+    Rshift,
+    And,
+    Xor,
+    Or,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum UnaryOperator {
+    Not,
+    Invert,
+    Minus,
+    Plus,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum Varargs {
+    None,
+    Unnamed,
+    Named(String),
+}
+
+/*
+Maintain a stack of blocks on the VM.
+pub enum BlockType {
+    Loop,
+    Except,
+}
+*/
+
+impl CodeObject {
+    pub fn new(
+        arg_names: Vec<String>,
+        varargs: Varargs,
+        kwonlyarg_names: Vec<String>,
+        varkeywords: Varargs,
+        source_path: String,
+        first_line_number: usize,
+        obj_name: String,
+    ) -> CodeObject {
+        CodeObject {
+            instructions: Vec::new(),
+            label_map: HashMap::new(),
+            locations: Vec::new(),
+            arg_names,
+            varargs,
+            kwonlyarg_names,
+            varkeywords,
+            source_path,
+            first_line_number,
+            obj_name,
+            is_generator: false,
+        }
+    }
+
+    pub fn get_constants(&self) -> impl Iterator<Item = &Constant> {
+        self.instructions.iter().filter_map(|x| {
+            if let Instruction::LoadConst { value } = x {
+                Some(value)
+            } else {
+                None
+            }
+        })
+    }
+}
+
+impl fmt::Display for CodeObject {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let label_targets: HashSet<&usize> = self.label_map.values().collect();
+        for (offset, instruction) in self.instructions.iter().enumerate() {
+            let arrow = if label_targets.contains(&offset) {
+                ">>"
+            } else {
+                "  "
+            };
+            write!(f, "          {} {:5} ", arrow, offset)?;
+            instruction.fmt_dis(f, &self.label_map)?;
+        }
+        Ok(())
+    }
+}
+
+impl Instruction {
+    fn fmt_dis(&self, f: &mut fmt::Formatter, label_map: &HashMap<Label, usize>) -> fmt::Result {
+        macro_rules! w {
+            ($variant:ident) => {
+                write!(f, "{:20}\n", stringify!($variant))
+            };
+            ($variant:ident, $var:expr) => {
+                write!(f, "{:20} ({})\n", stringify!($variant), $var)
+            };
+            ($variant:ident, $var1:expr, $var2:expr) => {
+                write!(f, "{:20} ({}, {})\n", stringify!($variant), $var1, $var2)
+            };
+        }
+
+        match self {
+            Import { name, symbol } => w!(Import, name, format!("{:?}", symbol)),
+            ImportStar { name } => w!(ImportStar, name),
+            LoadName { name, scope } => w!(LoadName, name, format!("{:?}", scope)),
+            StoreName { name, scope } => w!(StoreName, name, format!("{:?}", scope)),
+            DeleteName { name } => w!(DeleteName, name),
+            StoreSubscript => w!(StoreSubscript),
+            DeleteSubscript => w!(DeleteSubscript),
+            StoreAttr { name } => w!(StoreAttr, name),
+            DeleteAttr { name } => w!(DeleteAttr, name),
+            LoadConst { value } => w!(LoadConst, value),
+            UnaryOperation { op } => w!(UnaryOperation, format!("{:?}", op)),
+            BinaryOperation { op, inplace } => w!(BinaryOperation, format!("{:?}", op), inplace),
+            LoadAttr { name } => w!(LoadAttr, name),
+            CompareOperation { op } => w!(CompareOperation, format!("{:?}", op)),
+            Pop => w!(Pop),
+            Rotate { amount } => w!(Rotate, amount),
+            Duplicate => w!(Duplicate),
+            GetIter => w!(GetIter),
+            Pass => w!(Pass),
+            Continue => w!(Continue),
+            Break => w!(Break),
+            Jump { target } => w!(Jump, label_map[target]),
+            JumpIf { target } => w!(JumpIf, label_map[target]),
+            JumpIfFalse { target } => w!(JumpIfFalse, label_map[target]),
+            MakeFunction { flags } => w!(MakeFunction, format!("{:?}", flags)),
+            CallFunction { typ } => w!(CallFunction, format!("{:?}", typ)),
+            ForIter { target } => w!(ForIter, label_map[target]),
+            ReturnValue => w!(ReturnValue),
+            YieldValue => w!(YieldValue),
+            YieldFrom => w!(YieldFrom),
+            SetupLoop { start, end } => w!(SetupLoop, label_map[start], label_map[end]),
+            SetupExcept { handler } => w!(SetupExcept, handler),
+            SetupWith { end } => w!(SetupWith, end),
+            CleanupWith { end } => w!(CleanupWith, end),
+            PopBlock => w!(PopBlock),
+            Raise { argc } => w!(Raise, argc),
+            BuildString { size } => w!(BuildString, size),
+            BuildTuple { size, unpack } => w!(BuildTuple, size, unpack),
+            BuildList { size, unpack } => w!(BuildList, size, unpack),
+            BuildSet { size, unpack } => w!(BuildSet, size, unpack),
+            BuildMap { size, unpack } => w!(BuildMap, size, unpack),
+            BuildSlice { size } => w!(BuildSlice, size),
+            ListAppend { i } => w!(ListAppend, i),
+            SetAdd { i } => w!(SetAdd, i),
+            MapAdd { i } => w!(MapAdd, i),
+            PrintExpr => w!(PrintExpr),
+            LoadBuildClass => w!(LoadBuildClass),
+            UnpackSequence { size } => w!(UnpackSequence, size),
+            UnpackEx { before, after } => w!(UnpackEx, before, after),
+            Unpack => w!(Unpack),
+            FormatValue { spec, .. } => w!(FormatValue, spec), // TODO: write conversion
+            PopException => w!(PopException),
+        }
+    }
+}
+
+impl fmt::Display for Constant {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Constant::Integer { value } => write!(f, "{}", value),
+            Constant::Float { value } => write!(f, "{}", value),
+            Constant::Complex { value } => write!(f, "{}", value),
+            Constant::Boolean { value } => write!(f, "{}", value),
+            Constant::String { value } => write!(f, "{:?}", value),
+            Constant::Bytes { value } => write!(f, "{:?}", value),
+            Constant::Code { code } => write!(f, "{:?}", code),
+            Constant::Tuple { elements } => write!(
+                f,
+                "({})",
+                elements
+                    .iter()
+                    .map(|e| format!("{}", e))
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            ),
+            Constant::None => write!(f, "None"),
+            Constant::Ellipsis => write!(f, "Ellipsis"),
+        }
+    }
+}
+
+impl fmt::Debug for CodeObject {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "<code object {} at ??? file {:?}, line {}>",
+            self.obj_name, self.source_path, self.first_line_number
+        )
+    }
+}
+
+impl From<ast::Varargs> for Varargs {
+    fn from(varargs: ast::Varargs) -> Varargs {
+        match varargs {
+            ast::Varargs::None => Varargs::None,
+            ast::Varargs::Unnamed => Varargs::Unnamed,
+            ast::Varargs::Named(param) => Varargs::Named(param.arg),
+        }
+    }
+}
+
+impl<'a> From<&'a ast::Varargs> for Varargs {
+    fn from(varargs: &'a ast::Varargs) -> Varargs {
+        match varargs {
+            ast::Varargs::None => Varargs::None,
+            ast::Varargs::Unnamed => Varargs::Unnamed,
+            ast::Varargs::Named(ref param) => Varargs::Named(param.arg.clone()),
+        }
+    }
+}
diff --git a/src/compile.rs b/src/compile.rs
new file mode 100644
index 00000000..dec158e9
--- /dev/null
+++ b/src/compile.rs
@@ -0,0 +1,1874 @@
+//!
+//! Take an AST and transform it into bytecode
+//!
+//! Inspirational code:
+//!   https://github.com/python/cpython/blob/master/Python/compile.c
+//!   https://github.com/micropython/micropython/blob/master/py/compile.c
+
+use crate::bytecode::{self, CallType, CodeObject, Instruction, Varargs};
+use crate::error::{CompileError, CompileErrorType};
+use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolRole, SymbolScope};
+use num_complex::Complex64;
+use rustpython_parser::{ast, parser};
+
+struct Compiler {
+    code_object_stack: Vec<CodeObject>,
+    scope_stack: Vec<SymbolScope>,
+    nxt_label: usize,
+    source_path: Option<String>,
+    current_source_location: ast::Location,
+    current_qualified_path: Option<String>,
+    in_loop: bool,
+    in_function_def: bool,
+}
+
+/// Compile a given sourcecode into a bytecode object.
+pub fn compile(source: &str, mode: &Mode, source_path: String) -> Result<CodeObject, CompileError> {
+    let mut compiler = Compiler::new();
+    compiler.source_path = Some(source_path);
+    compiler.push_new_code_object("<module>".to_string());
+
+    match mode {
+        Mode::Exec => {
+            let ast = parser::parse_program(source)?;
+            let symbol_table = make_symbol_table(&ast)?;
+            compiler.compile_program(&ast, symbol_table)
+        }
+        Mode::Eval => {
+            let statement = parser::parse_statement(source)?;
+            let symbol_table = statements_to_symbol_table(&statement)?;
+            compiler.compile_statement_eval(&statement, symbol_table)
+        }
+        Mode::Single => {
+            let ast = parser::parse_program(source)?;
+            let symbol_table = make_symbol_table(&ast)?;
+            compiler.compile_program_single(&ast, symbol_table)
+        }
+    }?;
+
+    let code = compiler.pop_code_object();
+    trace!("Compilation completed: {:?}", code);
+    Ok(code)
+}
+
+pub enum Mode {
+    Exec,
+    Eval,
+    Single,
+}
+
+#[derive(Clone, Copy)]
+enum EvalContext {
+    Statement,
+    Expression,
+}
+
+type Label = usize;
+
+impl Compiler {
+    fn new() -> Self {
+        Compiler {
+            code_object_stack: Vec::new(),
+            scope_stack: Vec::new(),
+            nxt_label: 0,
+            source_path: None,
+            current_source_location: ast::Location::default(),
+            current_qualified_path: None,
+            in_loop: false,
+            in_function_def: false,
+        }
+    }
+
+    fn push_new_code_object(&mut self, obj_name: String) {
+        let line_number = self.get_source_line_number();
+        self.code_object_stack.push(CodeObject::new(
+            Vec::new(),
+            Varargs::None,
+            Vec::new(),
+            Varargs::None,
+            self.source_path.clone().unwrap(),
+            line_number,
+            obj_name,
+        ));
+    }
+
+    fn pop_code_object(&mut self) -> CodeObject {
+        // self.scope_stack.pop().unwrap();
+        self.code_object_stack.pop().unwrap()
+    }
+
+    fn compile_program(
+        &mut self,
+        program: &ast::Program,
+        symbol_scope: SymbolScope,
+    ) -> Result<(), CompileError> {
+        let size_before = self.code_object_stack.len();
+        self.scope_stack.push(symbol_scope);
+        self.compile_statements(&program.statements)?;
+        assert!(self.code_object_stack.len() == size_before);
+
+        // Emit None at end:
+        self.emit(Instruction::LoadConst {
+            value: bytecode::Constant::None,
+        });
+        self.emit(Instruction::ReturnValue);
+        Ok(())
+    }
+
+    fn compile_program_single(
+        &mut self,
+        program: &ast::Program,
+        symbol_scope: SymbolScope,
+    ) -> Result<(), CompileError> {
+        self.scope_stack.push(symbol_scope);
+
+        let mut emitted_return = false;
+
+        for (i, statement) in program.statements.iter().enumerate() {
+            let is_last = i == program.statements.len() - 1;
+
+            if let ast::Statement::Expression { ref expression } = statement.node {
+                self.compile_expression(expression)?;
+
+                if is_last {
+                    self.emit(Instruction::Duplicate);
+                    self.emit(Instruction::PrintExpr);
+                    self.emit(Instruction::ReturnValue);
+                    emitted_return = true;
+                } else {
+                    self.emit(Instruction::PrintExpr);
+                }
+            } else {
+                self.compile_statement(&statement)?;
+            }
+        }
+
+        if !emitted_return {
+            self.emit(Instruction::LoadConst {
+                value: bytecode::Constant::None,
+            });
+            self.emit(Instruction::ReturnValue);
+        }
+
+        Ok(())
+    }
+
+    // Compile statement in eval mode:
+    fn compile_statement_eval(
+        &mut self,
+        statements: &[ast::LocatedStatement],
+        symbol_table: SymbolScope,
+    ) -> Result<(), CompileError> {
+        self.scope_stack.push(symbol_table);
+        for statement in statements {
+            if let ast::Statement::Expression { ref expression } = statement.node {
+                self.compile_expression(expression)?;
+            } else {
+                return Err(CompileError {
+                    error: CompileErrorType::ExpectExpr,
+                    location: statement.location.clone(),
+                });
+            }
+        }
+        self.emit(Instruction::ReturnValue);
+        Ok(())
+    }
+
+    fn compile_statements(
+        &mut self,
+        statements: &[ast::LocatedStatement],
+    ) -> Result<(), CompileError> {
+        for statement in statements {
+            self.compile_statement(statement)?
+        }
+        Ok(())
+    }
+
+    fn scope_for_name(&self, name: &str) -> bytecode::NameScope {
+        let role = self.lookup_name(name);
+        match role {
+            SymbolRole::Global => bytecode::NameScope::Global,
+            SymbolRole::Nonlocal => bytecode::NameScope::NonLocal,
+            _ => bytecode::NameScope::Local,
+        }
+    }
+
+    fn load_name(&mut self, name: &str) {
+        let scope = self.scope_for_name(name);
+        self.emit(Instruction::LoadName {
+            name: name.to_string(),
+            scope,
+        });
+    }
+
+    fn store_name(&mut self, name: &str) {
+        let scope = self.scope_for_name(name);
+        self.emit(Instruction::StoreName {
+            name: name.to_string(),
+            scope,
+        });
+    }
+
+    fn compile_statement(&mut self, statement: &ast::LocatedStatement) -> Result<(), CompileError> {
+        trace!("Compiling {:?}", statement);
+        self.set_source_location(&statement.location);
+
+        match &statement.node {
+            ast::Statement::Import { import_parts } => {
+                for ast::SingleImport {
+                    module,
+                    symbol,
+                    alias,
+                } in import_parts
+                {
+                    match symbol {
+                        Some(name) if name == "*" => {
+                            self.emit(Instruction::ImportStar {
+                                name: module.clone(),
+                            });
+                        }
+                        _ => {
+                            self.emit(Instruction::Import {
+                                name: module.clone(),
+                                symbol: symbol.clone(),
+                            });
+                            let name = match alias {
+                                Some(alias) => alias.clone(),
+                                None => match symbol {
+                                    Some(symbol) => symbol.clone(),
+                                    None => module.clone(),
+                                },
+                            };
+                            self.store_name(&name);
+                        }
+                    }
+                }
+            }
+            ast::Statement::Expression { expression } => {
+                self.compile_expression(expression)?;
+
+                // Pop result of stack, since we not use it:
+                self.emit(Instruction::Pop);
+            }
+            ast::Statement::Global { .. } | ast::Statement::Nonlocal { .. } => {
+                // Handled during symbol table construction.
+            }
+            ast::Statement::If { test, body, orelse } => {
+                let end_label = self.new_label();
+                match orelse {
+                    None => {
+                        // Only if:
+                        self.compile_test(test, None, Some(end_label), EvalContext::Statement)?;
+                        self.compile_statements(body)?;
+                        self.set_label(end_label);
+                    }
+                    Some(statements) => {
+                        // if - else:
+                        let else_label = self.new_label();
+                        self.compile_test(test, None, Some(else_label), EvalContext::Statement)?;
+                        self.compile_statements(body)?;
+                        self.emit(Instruction::Jump { target: end_label });
+
+                        // else:
+                        self.set_label(else_label);
+                        self.compile_statements(statements)?;
+                    }
+                }
+                self.set_label(end_label);
+            }
+            ast::Statement::While { test, body, orelse } => {
+                let start_label = self.new_label();
+                let else_label = self.new_label();
+                let end_label = self.new_label();
+                self.emit(Instruction::SetupLoop {
+                    start: start_label,
+                    end: end_label,
+                });
+
+                self.set_label(start_label);
+
+                self.compile_test(test, None, Some(else_label), EvalContext::Statement)?;
+
+                let was_in_loop = self.in_loop;
+                self.in_loop = true;
+                self.compile_statements(body)?;
+                self.in_loop = was_in_loop;
+                self.emit(Instruction::Jump {
+                    target: start_label,
+                });
+                self.set_label(else_label);
+                self.emit(Instruction::PopBlock);
+                if let Some(orelse) = orelse {
+                    self.compile_statements(orelse)?;
+                }
+                self.set_label(end_label);
+            }
+            ast::Statement::With { items, body } => {
+                let end_label = self.new_label();
+                for item in items {
+                    self.compile_expression(&item.context_expr)?;
+                    self.emit(Instruction::SetupWith { end: end_label });
+                    match &item.optional_vars {
+                        Some(var) => {
+                            self.compile_store(var)?;
+                        }
+                        None => {
+                            self.emit(Instruction::Pop);
+                        }
+                    }
+                }
+
+                self.compile_statements(body)?;
+                for _ in 0..items.len() {
+                    self.emit(Instruction::CleanupWith { end: end_label });
+                }
+                self.set_label(end_label);
+            }
+            ast::Statement::For {
+                target,
+                iter,
+                body,
+                orelse,
+            } => self.compile_for(target, iter, body, orelse)?,
+            ast::Statement::AsyncFor { .. } => {
+                unimplemented!("async for");
+            }
+            ast::Statement::Raise { exception, cause } => match exception {
+                Some(value) => {
+                    self.compile_expression(value)?;
+                    match cause {
+                        Some(cause) => {
+                            self.compile_expression(cause)?;
+                            self.emit(Instruction::Raise { argc: 2 });
+                        }
+                        None => {
+                            self.emit(Instruction::Raise { argc: 1 });
+                        }
+                    }
+                }
+                None => {
+                    self.emit(Instruction::Raise { argc: 0 });
+                }
+            },
+            ast::Statement::Try {
+                body,
+                handlers,
+                orelse,
+                finalbody,
+            } => self.compile_try_statement(body, handlers, orelse, finalbody)?,
+            ast::Statement::FunctionDef {
+                name,
+                args,
+                body,
+                decorator_list,
+                returns,
+            } => self.compile_function_def(name, args, body, decorator_list, returns)?,
+            ast::Statement::AsyncFunctionDef { .. } => {
+                unimplemented!("async def");
+            }
+            ast::Statement::ClassDef {
+                name,
+                body,
+                bases,
+                keywords,
+                decorator_list,
+            } => self.compile_class_def(name, body, bases, keywords, decorator_list)?,
+            ast::Statement::Assert { test, msg } => {
+                // TODO: if some flag, ignore all assert statements!
+
+                let end_label = self.new_label();
+                self.compile_test(test, Some(end_label), None, EvalContext::Statement)?;
+                self.emit(Instruction::LoadName {
+                    name: String::from("AssertionError"),
+                    scope: bytecode::NameScope::Local,
+                });
+                match msg {
+                    Some(e) => {
+                        self.compile_expression(e)?;
+                        self.emit(Instruction::CallFunction {
+                            typ: CallType::Positional(1),
+                        });
+                    }
+                    None => {
+                        self.emit(Instruction::CallFunction {
+                            typ: CallType::Positional(0),
+                        });
+                    }
+                }
+                self.emit(Instruction::Raise { argc: 1 });
+                self.set_label(end_label);
+            }
+            ast::Statement::Break => {
+                if !self.in_loop {
+                    return Err(CompileError {
+                        error: CompileErrorType::InvalidBreak,
+                        location: statement.location.clone(),
+                    });
+                }
+                self.emit(Instruction::Break);
+            }
+            ast::Statement::Continue => {
+                if !self.in_loop {
+                    return Err(CompileError {
+                        error: CompileErrorType::InvalidContinue,
+                        location: statement.location.clone(),
+                    });
+                }
+                self.emit(Instruction::Continue);
+            }
+            ast::Statement::Return { value } => {
+                if !self.in_function_def {
+                    return Err(CompileError {
+                        error: CompileErrorType::InvalidReturn,
+                        location: statement.location.clone(),
+                    });
+                }
+                match value {
+                    Some(v) => {
+                        self.compile_expression(v)?;
+                    }
+                    None => {
+                        self.emit(Instruction::LoadConst {
+                            value: bytecode::Constant::None,
+                        });
+                    }
+                }
+
+                self.emit(Instruction::ReturnValue);
+            }
+            ast::Statement::Assign { targets, value } => {
+                self.compile_expression(value)?;
+
+                for (i, target) in targets.iter().enumerate() {
+                    if i + 1 != targets.len() {
+                        self.emit(Instruction::Duplicate);
+                    }
+                    self.compile_store(target)?;
+                }
+            }
+            ast::Statement::AugAssign { target, op, value } => {
+                self.compile_expression(target)?;
+                self.compile_expression(value)?;
+
+                // Perform operation:
+                self.compile_op(op, true);
+                self.compile_store(target)?;
+            }
+            ast::Statement::Delete { targets } => {
+                for target in targets {
+                    self.compile_delete(target)?;
+                }
+            }
+            ast::Statement::Pass => {
+                self.emit(Instruction::Pass);
+            }
+        }
+        Ok(())
+    }
+
+    fn compile_delete(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
+        match expression {
+            ast::Expression::Identifier { name } => {
+                self.emit(Instruction::DeleteName {
+                    name: name.to_string(),
+                });
+            }
+            ast::Expression::Attribute { value, name } => {
+                self.compile_expression(value)?;
+                self.emit(Instruction::DeleteAttr {
+                    name: name.to_string(),
+                });
+            }
+            ast::Expression::Subscript { a, b } => {
+                self.compile_expression(a)?;
+                self.compile_expression(b)?;
+                self.emit(Instruction::DeleteSubscript);
+            }
+            ast::Expression::Tuple { elements } => {
+                for element in elements {
+                    self.compile_delete(element)?;
+                }
+            }
+            _ => {
+                return Err(CompileError {
+                    error: CompileErrorType::Delete(expression.name()),
+                    location: self.current_source_location.clone(),
+                });
+            }
+        }
+        Ok(())
+    }
+
+    fn enter_function(
+        &mut self,
+        name: &str,
+        args: &ast::Parameters,
+    ) -> Result<bytecode::FunctionOpArg, CompileError> {
+        let have_defaults = !args.defaults.is_empty();
+        if have_defaults {
+            // Construct a tuple:
+            let size = args.defaults.len();
+            for element in &args.defaults {
+                self.compile_expression(element)?;
+            }
+            self.emit(Instruction::BuildTuple {
+                size,
+                unpack: false,
+            });
+        }
+
+        let mut num_kw_only_defaults = 0;
+        for (kw, default) in args.kwonlyargs.iter().zip(&args.kw_defaults) {
+            if let Some(default) = default {
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::String {
+                        value: kw.arg.clone(),
+                    },
+                });
+                self.compile_expression(default)?;
+                num_kw_only_defaults += 1;
+            }
+        }
+        if num_kw_only_defaults > 0 {
+            self.emit(Instruction::BuildMap {
+                size: num_kw_only_defaults,
+                unpack: false,
+            });
+        }
+
+        let line_number = self.get_source_line_number();
+        self.code_object_stack.push(CodeObject::new(
+            args.args.iter().map(|a| a.arg.clone()).collect(),
+            Varargs::from(&args.vararg),
+            args.kwonlyargs.iter().map(|a| a.arg.clone()).collect(),
+            Varargs::from(&args.kwarg),
+            self.source_path.clone().unwrap(),
+            line_number,
+            name.to_string(),
+        ));
+        self.enter_scope();
+
+        let mut flags = bytecode::FunctionOpArg::empty();
+        if have_defaults {
+            flags |= bytecode::FunctionOpArg::HAS_DEFAULTS;
+        }
+        if num_kw_only_defaults > 0 {
+            flags |= bytecode::FunctionOpArg::HAS_KW_ONLY_DEFAULTS;
+        }
+
+        Ok(flags)
+    }
+
+    fn prepare_decorators(
+        &mut self,
+        decorator_list: &[ast::Expression],
+    ) -> Result<(), CompileError> {
+        for decorator in decorator_list {
+            self.compile_expression(decorator)?;
+        }
+        Ok(())
+    }
+
+    fn apply_decorators(&mut self, decorator_list: &[ast::Expression]) {
+        // Apply decorators:
+        for _ in decorator_list {
+            self.emit(Instruction::CallFunction {
+                typ: CallType::Positional(1),
+            });
+        }
+    }
+
+    fn compile_try_statement(
+        &mut self,
+        body: &[ast::LocatedStatement],
+        handlers: &[ast::ExceptHandler],
+        orelse: &Option<Vec<ast::LocatedStatement>>,
+        finalbody: &Option<Vec<ast::LocatedStatement>>,
+    ) -> Result<(), CompileError> {
+        let mut handler_label = self.new_label();
+        let finally_label = self.new_label();
+        let else_label = self.new_label();
+        // try:
+        self.emit(Instruction::SetupExcept {
+            handler: handler_label,
+        });
+        self.compile_statements(body)?;
+        self.emit(Instruction::PopBlock);
+        self.emit(Instruction::Jump { target: else_label });
+
+        // except handlers:
+        self.set_label(handler_label);
+        // Exception is on top of stack now
+        handler_label = self.new_label();
+        for handler in handlers {
+            // If we gave a typ,
+            // check if this handler can handle the exception:
+            if let Some(exc_type) = &handler.typ {
+                // Duplicate exception for test:
+                self.emit(Instruction::Duplicate);
+
+                // Check exception type:
+                self.emit(Instruction::LoadName {
+                    name: String::from("isinstance"),
+                    scope: bytecode::NameScope::Local,
+                });
+                self.emit(Instruction::Rotate { amount: 2 });
+                self.compile_expression(exc_type)?;
+                self.emit(Instruction::CallFunction {
+                    typ: CallType::Positional(2),
+                });
+
+                // We cannot handle this exception type:
+                self.emit(Instruction::JumpIfFalse {
+                    target: handler_label,
+                });
+
+                // We have a match, store in name (except x as y)
+                if let Some(alias) = &handler.name {
+                    self.store_name(alias);
+                } else {
+                    // Drop exception from top of stack:
+                    self.emit(Instruction::Pop);
+                }
+            } else {
+                // Catch all!
+                // Drop exception from top of stack:
+                self.emit(Instruction::Pop);
+            }
+
+            // Handler code:
+            self.compile_statements(&handler.body)?;
+            self.emit(Instruction::PopException);
+            self.emit(Instruction::Jump {
+                target: finally_label,
+            });
+
+            // Emit a new label for the next handler
+            self.set_label(handler_label);
+            handler_label = self.new_label();
+        }
+        self.emit(Instruction::Jump {
+            target: handler_label,
+        });
+        self.set_label(handler_label);
+        // If code flows here, we have an unhandled exception,
+        // emit finally code and raise again!
+        // Duplicate finally code here:
+        // TODO: this bytecode is now duplicate, could this be
+        // improved?
+        if let Some(statements) = finalbody {
+            self.compile_statements(statements)?;
+        }
+        self.emit(Instruction::Raise { argc: 0 });
+
+        // We successfully ran the try block:
+        // else:
+        self.set_label(else_label);
+        if let Some(statements) = orelse {
+            self.compile_statements(statements)?;
+        }
+
+        // finally:
+        self.set_label(finally_label);
+        if let Some(statements) = finalbody {
+            self.compile_statements(statements)?;
+        }
+        // unimplemented!();
+        Ok(())
+    }
+
+    fn compile_function_def(
+        &mut self,
+        name: &str,
+        args: &ast::Parameters,
+        body: &[ast::LocatedStatement],
+        decorator_list: &[ast::Expression],
+        returns: &Option<ast::Expression>, // TODO: use type hint somehow..
+    ) -> Result<(), CompileError> {
+        // Create bytecode for this function:
+        // remember to restore self.in_loop to the original after the function is compiled
+        let was_in_loop = self.in_loop;
+        let was_in_function_def = self.in_function_def;
+        self.in_loop = false;
+        self.in_function_def = true;
+
+        let old_qualified_path = self.current_qualified_path.clone();
+        let qualified_name = self.create_qualified_name(name, "");
+        self.current_qualified_path = Some(self.create_qualified_name(name, ".<locals>"));
+
+        self.prepare_decorators(decorator_list)?;
+
+        let mut flags = self.enter_function(name, args)?;
+
+        let (new_body, doc_str) = get_doc(body);
+
+        self.compile_statements(new_body)?;
+
+        // Emit None at end:
+        self.emit(Instruction::LoadConst {
+            value: bytecode::Constant::None,
+        });
+        self.emit(Instruction::ReturnValue);
+        let code = self.pop_code_object();
+        self.leave_scope();
+
+        // Prepare type annotations:
+        let mut num_annotations = 0;
+
+        // Return annotation:
+        if let Some(annotation) = returns {
+            // key:
+            self.emit(Instruction::LoadConst {
+                value: bytecode::Constant::String {
+                    value: "return".to_string(),
+                },
+            });
+            // value:
+            self.compile_expression(annotation)?;
+            num_annotations += 1;
+        }
+
+        for arg in args.args.iter() {
+            if let Some(annotation) = &arg.annotation {
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::String {
+                        value: arg.arg.to_string(),
+                    },
+                });
+                self.compile_expression(&annotation)?;
+                num_annotations += 1;
+            }
+        }
+
+        if num_annotations > 0 {
+            flags |= bytecode::FunctionOpArg::HAS_ANNOTATIONS;
+            self.emit(Instruction::BuildMap {
+                size: num_annotations,
+                unpack: false,
+            });
+        }
+
+        self.emit(Instruction::LoadConst {
+            value: bytecode::Constant::Code {
+                code: Box::new(code),
+            },
+        });
+        self.emit(Instruction::LoadConst {
+            value: bytecode::Constant::String {
+                value: qualified_name,
+            },
+        });
+
+        // Turn code object into function object:
+        self.emit(Instruction::MakeFunction { flags });
+        self.store_docstring(doc_str);
+        self.apply_decorators(decorator_list);
+
+        self.store_name(name);
+
+        self.current_qualified_path = old_qualified_path;
+        self.in_loop = was_in_loop;
+        self.in_function_def = was_in_function_def;
+        Ok(())
+    }
+
+    fn compile_class_def(
+        &mut self,
+        name: &str,
+        body: &[ast::LocatedStatement],
+        bases: &[ast::Expression],
+        keywords: &[ast::Keyword],
+        decorator_list: &[ast::Expression],
+    ) -> Result<(), CompileError> {
+        let was_in_loop = self.in_loop;
+        self.in_loop = false;
+
+        let old_qualified_path = self.current_qualified_path.clone();
+        let qualified_name = self.create_qualified_name(name, "");
+        self.current_qualified_path = Some(qualified_name.clone());
+
+        self.prepare_decorators(decorator_list)?;
+        self.emit(Instruction::LoadBuildClass);
+        let line_number = self.get_source_line_number();
+        self.code_object_stack.push(CodeObject::new(
+            vec![],
+            Varargs::None,
+            vec![],
+            Varargs::None,
+            self.source_path.clone().unwrap(),
+            line_number,
+            name.to_string(),
+        ));
+        self.enter_scope();
+
+        let (new_body, doc_str) = get_doc(body);
+
+        self.emit(Instruction::LoadName {
+            name: "__name__".to_string(),
+            scope: bytecode::NameScope::Local,
+        });
+        self.emit(Instruction::StoreName {
+            name: "__module__".to_string(),
+            scope: bytecode::NameScope::Local,
+        });
+        self.compile_statements(new_body)?;
+        self.emit(Instruction::LoadConst {
+            value: bytecode::Constant::None,
+        });
+        self.emit(Instruction::ReturnValue);
+
+        let code = self.pop_code_object();
+        self.leave_scope();
+
+        self.emit(Instruction::LoadConst {
+            value: bytecode::Constant::Code {
+                code: Box::new(code),
+            },
+        });
+        self.emit(Instruction::LoadConst {
+            value: bytecode::Constant::String {
+                value: name.to_string(),
+            },
+        });
+
+        // Turn code object into function object:
+        self.emit(Instruction::MakeFunction {
+            flags: bytecode::FunctionOpArg::empty(),
+        });
+
+        self.emit(Instruction::LoadConst {
+            value: bytecode::Constant::String {
+                value: qualified_name,
+            },
+        });
+
+        for base in bases {
+            self.compile_expression(base)?;
+        }
+
+        if !keywords.is_empty() {
+            let mut kwarg_names = vec![];
+            for keyword in keywords {
+                if let Some(name) = &keyword.name {
+                    kwarg_names.push(bytecode::Constant::String {
+                        value: name.to_string(),
+                    });
+                } else {
+                    // This means **kwargs!
+                    panic!("name must be set");
+                }
+                self.compile_expression(&keyword.value)?;
+            }
+
+            self.emit(Instruction::LoadConst {
+                value: bytecode::Constant::Tuple {
+                    elements: kwarg_names,
+                },
+            });
+            self.emit(Instruction::CallFunction {
+                typ: CallType::Keyword(2 + keywords.len() + bases.len()),
+            });
+        } else {
+            self.emit(Instruction::CallFunction {
+                typ: CallType::Positional(2 + bases.len()),
+            });
+        }
+
+        self.store_docstring(doc_str);
+        self.apply_decorators(decorator_list);
+
+        self.store_name(name);
+        self.current_qualified_path = old_qualified_path;
+        self.in_loop = was_in_loop;
+        Ok(())
+    }
+
+    fn store_docstring(&mut self, doc_str: Option<String>) {
+        if let Some(doc_string) = doc_str {
+            // Duplicate top of stack (the function or class object)
+            self.emit(Instruction::Duplicate);
+
+            // Doc string value:
+            self.emit(Instruction::LoadConst {
+                value: bytecode::Constant::String {
+                    value: doc_string.to_string(),
+                },
+            });
+
+            self.emit(Instruction::Rotate { amount: 2 });
+            self.emit(Instruction::StoreAttr {
+                name: "__doc__".to_string(),
+            });
+        }
+    }
+
+    fn compile_for(
+        &mut self,
+        target: &ast::Expression,
+        iter: &ast::Expression,
+        body: &[ast::LocatedStatement],
+        orelse: &Option<Vec<ast::LocatedStatement>>,
+    ) -> Result<(), CompileError> {
+        // Start loop
+        let start_label = self.new_label();
+        let else_label = self.new_label();
+        let end_label = self.new_label();
+        self.emit(Instruction::SetupLoop {
+            start: start_label,
+            end: end_label,
+        });
+
+        // The thing iterated:
+        self.compile_expression(iter)?;
+
+        // Retrieve Iterator
+        self.emit(Instruction::GetIter);
+
+        self.set_label(start_label);
+        self.emit(Instruction::ForIter { target: else_label });
+
+        // Start of loop iteration, set targets:
+        self.compile_store(target)?;
+
+        let was_in_loop = self.in_loop;
+        self.in_loop = true;
+        self.compile_statements(body)?;
+        self.in_loop = was_in_loop;
+
+        self.emit(Instruction::Jump {
+            target: start_label,
+        });
+        self.set_label(else_label);
+        self.emit(Instruction::PopBlock);
+        if let Some(orelse) = orelse {
+            self.compile_statements(orelse)?;
+        }
+        self.set_label(end_label);
+        Ok(())
+    }
+
+    fn compile_chained_comparison(
+        &mut self,
+        vals: &[ast::Expression],
+        ops: &[ast::Comparison],
+    ) -> Result<(), CompileError> {
+        assert!(!ops.is_empty());
+        assert_eq!(vals.len(), ops.len() + 1);
+
+        let to_operator = |op: &ast::Comparison| match op {
+            ast::Comparison::Equal => bytecode::ComparisonOperator::Equal,
+            ast::Comparison::NotEqual => bytecode::ComparisonOperator::NotEqual,
+            ast::Comparison::Less => bytecode::ComparisonOperator::Less,
+            ast::Comparison::LessOrEqual => bytecode::ComparisonOperator::LessOrEqual,
+            ast::Comparison::Greater => bytecode::ComparisonOperator::Greater,
+            ast::Comparison::GreaterOrEqual => bytecode::ComparisonOperator::GreaterOrEqual,
+            ast::Comparison::In => bytecode::ComparisonOperator::In,
+            ast::Comparison::NotIn => bytecode::ComparisonOperator::NotIn,
+            ast::Comparison::Is => bytecode::ComparisonOperator::Is,
+            ast::Comparison::IsNot => bytecode::ComparisonOperator::IsNot,
+        };
+
+        // a == b == c == d
+        // compile into (pseudocode):
+        // result = a == b
+        // if result:
+        //   result = b == c
+        //   if result:
+        //     result = c == d
+
+        // initialize lhs outside of loop
+        self.compile_expression(&vals[0])?;
+
+        let break_label = self.new_label();
+        let last_label = self.new_label();
+
+        // for all comparisons except the last (as the last one doesn't need a conditional jump)
+        let ops_slice = &ops[0..ops.len()];
+        let vals_slice = &vals[1..ops.len()];
+        for (op, val) in ops_slice.iter().zip(vals_slice.iter()) {
+            self.compile_expression(val)?;
+            // store rhs for the next comparison in chain
+            self.emit(Instruction::Duplicate);
+            self.emit(Instruction::Rotate { amount: 3 });
+
+            self.emit(Instruction::CompareOperation {
+                op: to_operator(op),
+            });
+
+            // if comparison result is false, we break with this value; if true, try the next one.
+            // (CPython compresses these three opcodes into JUMP_IF_FALSE_OR_POP)
+            self.emit(Instruction::Duplicate);
+            self.emit(Instruction::JumpIfFalse {
+                target: break_label,
+            });
+            self.emit(Instruction::Pop);
+        }
+
+        // handle the last comparison
+        self.compile_expression(vals.last().unwrap())?;
+        self.emit(Instruction::CompareOperation {
+            op: to_operator(ops.last().unwrap()),
+        });
+        self.emit(Instruction::Jump { target: last_label });
+
+        // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs.
+        self.set_label(break_label);
+        self.emit(Instruction::Rotate { amount: 2 });
+        self.emit(Instruction::Pop);
+
+        self.set_label(last_label);
+        Ok(())
+    }
+
+    fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> {
+        match target {
+            ast::Expression::Identifier { name } => {
+                self.store_name(name);
+            }
+            ast::Expression::Subscript { a, b } => {
+                self.compile_expression(a)?;
+                self.compile_expression(b)?;
+                self.emit(Instruction::StoreSubscript);
+            }
+            ast::Expression::Attribute { value, name } => {
+                self.compile_expression(value)?;
+                self.emit(Instruction::StoreAttr {
+                    name: name.to_string(),
+                });
+            }
+            ast::Expression::List { elements } | ast::Expression::Tuple { elements } => {
+                let mut seen_star = false;
+
+                // Scan for star args:
+                for (i, element) in elements.iter().enumerate() {
+                    if let ast::Expression::Starred { .. } = element {
+                        if seen_star {
+                            return Err(CompileError {
+                                error: CompileErrorType::StarArgs,
+                                location: self.current_source_location.clone(),
+                            });
+                        } else {
+                            seen_star = true;
+                            self.emit(Instruction::UnpackEx {
+                                before: i,
+                                after: elements.len() - i - 1,
+                            });
+                        }
+                    }
+                }
+
+                if !seen_star {
+                    self.emit(Instruction::UnpackSequence {
+                        size: elements.len(),
+                    });
+                }
+
+                for element in elements {
+                    if let ast::Expression::Starred { value } = element {
+                        self.compile_store(value)?;
+                    } else {
+                        self.compile_store(element)?;
+                    }
+                }
+            }
+            _ => {
+                return Err(CompileError {
+                    error: CompileErrorType::Assign(target.name()),
+                    location: self.current_source_location.clone(),
+                });
+            }
+        }
+
+        Ok(())
+    }
+
+    fn compile_op(&mut self, op: &ast::Operator, inplace: bool) {
+        let i = match op {
+            ast::Operator::Add => bytecode::BinaryOperator::Add,
+            ast::Operator::Sub => bytecode::BinaryOperator::Subtract,
+            ast::Operator::Mult => bytecode::BinaryOperator::Multiply,
+            ast::Operator::MatMult => bytecode::BinaryOperator::MatrixMultiply,
+            ast::Operator::Div => bytecode::BinaryOperator::Divide,
+            ast::Operator::FloorDiv => bytecode::BinaryOperator::FloorDivide,
+            ast::Operator::Mod => bytecode::BinaryOperator::Modulo,
+            ast::Operator::Pow => bytecode::BinaryOperator::Power,
+            ast::Operator::LShift => bytecode::BinaryOperator::Lshift,
+            ast::Operator::RShift => bytecode::BinaryOperator::Rshift,
+            ast::Operator::BitOr => bytecode::BinaryOperator::Or,
+            ast::Operator::BitXor => bytecode::BinaryOperator::Xor,
+            ast::Operator::BitAnd => bytecode::BinaryOperator::And,
+        };
+        self.emit(Instruction::BinaryOperation { op: i, inplace });
+    }
+
+    fn compile_test(
+        &mut self,
+        expression: &ast::Expression,
+        true_label: Option<Label>,
+        false_label: Option<Label>,
+        context: EvalContext,
+    ) -> Result<(), CompileError> {
+        // Compile expression for test, and jump to label if false
+        match expression {
+            ast::Expression::BoolOp { a, op, b } => match op {
+                ast::BooleanOperator::And => {
+                    let f = false_label.unwrap_or_else(|| self.new_label());
+                    self.compile_test(a, None, Some(f), context)?;
+                    self.compile_test(b, true_label, false_label, context)?;
+                    if false_label.is_none() {
+                        self.set_label(f);
+                    }
+                }
+                ast::BooleanOperator::Or => {
+                    let t = true_label.unwrap_or_else(|| self.new_label());
+                    self.compile_test(a, Some(t), None, context)?;
+                    self.compile_test(b, true_label, false_label, context)?;
+                    if true_label.is_none() {
+                        self.set_label(t);
+                    }
+                }
+            },
+            _ => {
+                self.compile_expression(expression)?;
+                match context {
+                    EvalContext::Statement => {
+                        if let Some(true_label) = true_label {
+                            self.emit(Instruction::JumpIf { target: true_label });
+                        }
+                        if let Some(false_label) = false_label {
+                            self.emit(Instruction::JumpIfFalse {
+                                target: false_label,
+                            });
+                        }
+                    }
+                    EvalContext::Expression => {
+                        if let Some(true_label) = true_label {
+                            self.emit(Instruction::Duplicate);
+                            self.emit(Instruction::JumpIf { target: true_label });
+                            self.emit(Instruction::Pop);
+                        }
+                        if let Some(false_label) = false_label {
+                            self.emit(Instruction::Duplicate);
+                            self.emit(Instruction::JumpIfFalse {
+                                target: false_label,
+                            });
+                            self.emit(Instruction::Pop);
+                        }
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn compile_expression(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
+        trace!("Compiling {:?}", expression);
+        match expression {
+            ast::Expression::Call {
+                function,
+                args,
+                keywords,
+            } => self.compile_call(function, args, keywords)?,
+            ast::Expression::BoolOp { .. } => {
+                self.compile_test(expression, None, None, EvalContext::Expression)?
+            }
+            ast::Expression::Binop { a, op, b } => {
+                self.compile_expression(a)?;
+                self.compile_expression(b)?;
+
+                // Perform operation:
+                self.compile_op(op, false);
+            }
+            ast::Expression::Subscript { a, b } => {
+                self.compile_expression(a)?;
+                self.compile_expression(b)?;
+                self.emit(Instruction::BinaryOperation {
+                    op: bytecode::BinaryOperator::Subscript,
+                    inplace: false,
+                });
+            }
+            ast::Expression::Unop { op, a } => {
+                self.compile_expression(a)?;
+
+                // Perform operation:
+                let i = match op {
+                    ast::UnaryOperator::Pos => bytecode::UnaryOperator::Plus,
+                    ast::UnaryOperator::Neg => bytecode::UnaryOperator::Minus,
+                    ast::UnaryOperator::Not => bytecode::UnaryOperator::Not,
+                    ast::UnaryOperator::Inv => bytecode::UnaryOperator::Invert,
+                };
+                let i = Instruction::UnaryOperation { op: i };
+                self.emit(i);
+            }
+            ast::Expression::Attribute { value, name } => {
+                self.compile_expression(value)?;
+                self.emit(Instruction::LoadAttr {
+                    name: name.to_string(),
+                });
+            }
+            ast::Expression::Compare { vals, ops } => {
+                self.compile_chained_comparison(vals, ops)?;
+            }
+            ast::Expression::Number { value } => {
+                let const_value = match value {
+                    ast::Number::Integer { value } => bytecode::Constant::Integer {
+                        value: value.clone(),
+                    },
+                    ast::Number::Float { value } => bytecode::Constant::Float { value: *value },
+                    ast::Number::Complex { real, imag } => bytecode::Constant::Complex {
+                        value: Complex64::new(*real, *imag),
+                    },
+                };
+                self.emit(Instruction::LoadConst { value: const_value });
+            }
+            ast::Expression::List { elements } => {
+                let size = elements.len();
+                let must_unpack = self.gather_elements(elements)?;
+                self.emit(Instruction::BuildList {
+                    size,
+                    unpack: must_unpack,
+                });
+            }
+            ast::Expression::Tuple { elements } => {
+                let size = elements.len();
+                let must_unpack = self.gather_elements(elements)?;
+                self.emit(Instruction::BuildTuple {
+                    size,
+                    unpack: must_unpack,
+                });
+            }
+            ast::Expression::Set { elements } => {
+                let size = elements.len();
+                let must_unpack = self.gather_elements(elements)?;
+                self.emit(Instruction::BuildSet {
+                    size,
+                    unpack: must_unpack,
+                });
+            }
+            ast::Expression::Dict { elements } => {
+                let size = elements.len();
+                let has_double_star = elements.iter().any(|e| e.0.is_none());
+                for (key, value) in elements {
+                    if let Some(key) = key {
+                        self.compile_expression(key)?;
+                        self.compile_expression(value)?;
+                        if has_double_star {
+                            self.emit(Instruction::BuildMap {
+                                size: 1,
+                                unpack: false,
+                            });
+                        }
+                    } else {
+                        // dict unpacking
+                        self.compile_expression(value)?;
+                    }
+                }
+                self.emit(Instruction::BuildMap {
+                    size,
+                    unpack: has_double_star,
+                });
+            }
+            ast::Expression::Slice { elements } => {
+                let size = elements.len();
+                for element in elements {
+                    self.compile_expression(element)?;
+                }
+                self.emit(Instruction::BuildSlice { size });
+            }
+            ast::Expression::Yield { value } => {
+                if !self.in_function_def {
+                    return Err(CompileError {
+                        error: CompileErrorType::InvalidYield,
+                        location: self.current_source_location.clone(),
+                    });
+                }
+                self.mark_generator();
+                match value {
+                    Some(expression) => self.compile_expression(expression)?,
+                    None => self.emit(Instruction::LoadConst {
+                        value: bytecode::Constant::None,
+                    }),
+                };
+                self.emit(Instruction::YieldValue);
+            }
+            ast::Expression::Await { .. } => {
+                unimplemented!("await");
+            }
+            ast::Expression::YieldFrom { value } => {
+                self.mark_generator();
+                self.compile_expression(value)?;
+                self.emit(Instruction::GetIter);
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::None,
+                });
+                self.emit(Instruction::YieldFrom);
+            }
+            ast::Expression::True => {
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::Boolean { value: true },
+                });
+            }
+            ast::Expression::False => {
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::Boolean { value: false },
+                });
+            }
+            ast::Expression::None => {
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::None,
+                });
+            }
+            ast::Expression::Ellipsis => {
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::Ellipsis,
+                });
+            }
+            ast::Expression::String { value } => {
+                self.compile_string(value)?;
+            }
+            ast::Expression::Bytes { value } => {
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::Bytes {
+                        value: value.clone(),
+                    },
+                });
+            }
+            ast::Expression::Identifier { name } => {
+                self.load_name(name);
+            }
+            ast::Expression::Lambda { args, body } => {
+                let name = "<lambda>".to_string();
+                // no need to worry about the self.loop_depth because there are no loops in lambda expressions
+                let flags = self.enter_function(&name, args)?;
+                self.compile_expression(body)?;
+                self.emit(Instruction::ReturnValue);
+                let code = self.pop_code_object();
+                self.leave_scope();
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::Code {
+                        code: Box::new(code),
+                    },
+                });
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::String { value: name },
+                });
+                // Turn code object into function object:
+                self.emit(Instruction::MakeFunction { flags });
+            }
+            ast::Expression::Comprehension { kind, generators } => {
+                self.compile_comprehension(kind, generators)?;
+            }
+            ast::Expression::Starred { value } => {
+                self.compile_expression(value)?;
+                self.emit(Instruction::Unpack);
+                panic!("We should not just unpack a starred args, since the size is unknown.");
+            }
+            ast::Expression::IfExpression { test, body, orelse } => {
+                let no_label = self.new_label();
+                let end_label = self.new_label();
+                self.compile_test(test, None, Some(no_label), EvalContext::Expression)?;
+                self.compile_expression(body)?;
+                self.emit(Instruction::Jump { target: end_label });
+                self.set_label(no_label);
+                self.compile_expression(orelse)?;
+                self.set_label(end_label);
+            }
+        }
+        Ok(())
+    }
+
+    fn compile_call(
+        &mut self,
+        function: &ast::Expression,
+        args: &[ast::Expression],
+        keywords: &[ast::Keyword],
+    ) -> Result<(), CompileError> {
+        self.compile_expression(function)?;
+        let count = args.len() + keywords.len();
+
+        // Normal arguments:
+        let must_unpack = self.gather_elements(args)?;
+        let has_double_star = keywords.iter().any(|k| k.name.is_none());
+
+        if must_unpack || has_double_star {
+            // Create a tuple with positional args:
+            self.emit(Instruction::BuildTuple {
+                size: args.len(),
+                unpack: must_unpack,
+            });
+
+            // Create an optional map with kw-args:
+            if !keywords.is_empty() {
+                for keyword in keywords {
+                    if let Some(name) = &keyword.name {
+                        self.emit(Instruction::LoadConst {
+                            value: bytecode::Constant::String {
+                                value: name.to_string(),
+                            },
+                        });
+                        self.compile_expression(&keyword.value)?;
+                        if has_double_star {
+                            self.emit(Instruction::BuildMap {
+                                size: 1,
+                                unpack: false,
+                            });
+                        }
+                    } else {
+                        // This means **kwargs!
+                        self.compile_expression(&keyword.value)?;
+                    }
+                }
+
+                self.emit(Instruction::BuildMap {
+                    size: keywords.len(),
+                    unpack: has_double_star,
+                });
+
+                self.emit(Instruction::CallFunction {
+                    typ: CallType::Ex(true),
+                });
+            } else {
+                self.emit(Instruction::CallFunction {
+                    typ: CallType::Ex(false),
+                });
+            }
+        } else {
+            // Keyword arguments:
+            if !keywords.is_empty() {
+                let mut kwarg_names = vec![];
+                for keyword in keywords {
+                    if let Some(name) = &keyword.name {
+                        kwarg_names.push(bytecode::Constant::String {
+                            value: name.to_string(),
+                        });
+                    } else {
+                        // This means **kwargs!
+                        panic!("name must be set");
+                    }
+                    self.compile_expression(&keyword.value)?;
+                }
+
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::Tuple {
+                        elements: kwarg_names,
+                    },
+                });
+                self.emit(Instruction::CallFunction {
+                    typ: CallType::Keyword(count),
+                });
+            } else {
+                self.emit(Instruction::CallFunction {
+                    typ: CallType::Positional(count),
+                });
+            }
+        }
+        Ok(())
+    }
+
+    // Given a vector of expr / star expr generate code which gives either
+    // a list of expressions on the stack, or a list of tuples.
+    fn gather_elements(&mut self, elements: &[ast::Expression]) -> Result<bool, CompileError> {
+        // First determine if we have starred elements:
+        let has_stars = elements.iter().any(|e| {
+            if let ast::Expression::Starred { .. } = e {
+                true
+            } else {
+                false
+            }
+        });
+
+        for element in elements {
+            if let ast::Expression::Starred { value } = element {
+                self.compile_expression(value)?;
+            } else {
+                self.compile_expression(element)?;
+                if has_stars {
+                    self.emit(Instruction::BuildTuple {
+                        size: 1,
+                        unpack: false,
+                    });
+                }
+            }
+        }
+
+        Ok(has_stars)
+    }
+
+    fn compile_comprehension(
+        &mut self,
+        kind: &ast::ComprehensionKind,
+        generators: &[ast::Comprehension],
+    ) -> Result<(), CompileError> {
+        // We must have at least one generator:
+        assert!(!generators.is_empty());
+
+        let name = match kind {
+            ast::ComprehensionKind::GeneratorExpression { .. } => "<genexpr>",
+            ast::ComprehensionKind::List { .. } => "<listcomp>",
+            ast::ComprehensionKind::Set { .. } => "<setcomp>",
+            ast::ComprehensionKind::Dict { .. } => "<dictcomp>",
+        }
+        .to_string();
+
+        let line_number = self.get_source_line_number();
+        // Create magnificent function <listcomp>:
+        self.code_object_stack.push(CodeObject::new(
+            vec![".0".to_string()],
+            Varargs::None,
+            vec![],
+            Varargs::None,
+            self.source_path.clone().unwrap(),
+            line_number,
+            name.clone(),
+        ));
+
+        // Create empty object of proper type:
+        match kind {
+            ast::ComprehensionKind::GeneratorExpression { .. } => {}
+            ast::ComprehensionKind::List { .. } => {
+                self.emit(Instruction::BuildList {
+                    size: 0,
+                    unpack: false,
+                });
+            }
+            ast::ComprehensionKind::Set { .. } => {
+                self.emit(Instruction::BuildSet {
+                    size: 0,
+                    unpack: false,
+                });
+            }
+            ast::ComprehensionKind::Dict { .. } => {
+                self.emit(Instruction::BuildMap {
+                    size: 0,
+                    unpack: false,
+                });
+            }
+        }
+
+        let mut loop_labels = vec![];
+        for generator in generators {
+            if loop_labels.is_empty() {
+                // Load iterator onto stack (passed as first argument):
+                self.emit(Instruction::LoadName {
+                    name: String::from(".0"),
+                    scope: bytecode::NameScope::Local,
+                });
+            } else {
+                // Evaluate iterated item:
+                self.compile_expression(&generator.iter)?;
+
+                // Get iterator / turn item into an iterator
+                self.emit(Instruction::GetIter);
+            }
+
+            // Setup for loop:
+            let start_label = self.new_label();
+            let end_label = self.new_label();
+            loop_labels.push((start_label, end_label));
+            self.emit(Instruction::SetupLoop {
+                start: start_label,
+                end: end_label,
+            });
+            self.set_label(start_label);
+            self.emit(Instruction::ForIter { target: end_label });
+
+            self.compile_store(&generator.target)?;
+
+            // Now evaluate the ifs:
+            for if_condition in &generator.ifs {
+                self.compile_test(
+                    if_condition,
+                    None,
+                    Some(start_label),
+                    EvalContext::Statement,
+                )?
+            }
+        }
+
+        match kind {
+            ast::ComprehensionKind::GeneratorExpression { element } => {
+                self.compile_expression(element)?;
+                self.mark_generator();
+                self.emit(Instruction::YieldValue);
+                self.emit(Instruction::Pop);
+            }
+            ast::ComprehensionKind::List { element } => {
+                self.compile_expression(element)?;
+                self.emit(Instruction::ListAppend {
+                    i: 1 + generators.len(),
+                });
+            }
+            ast::ComprehensionKind::Set { element } => {
+                self.compile_expression(element)?;
+                self.emit(Instruction::SetAdd {
+                    i: 1 + generators.len(),
+                });
+            }
+            ast::ComprehensionKind::Dict { key, value } => {
+                self.compile_expression(value)?;
+                self.compile_expression(key)?;
+
+                self.emit(Instruction::MapAdd {
+                    i: 1 + generators.len(),
+                });
+            }
+        }
+
+        for (start_label, end_label) in loop_labels.iter().rev() {
+            // Repeat:
+            self.emit(Instruction::Jump {
+                target: *start_label,
+            });
+
+            // End of for loop:
+            self.set_label(*end_label);
+            self.emit(Instruction::PopBlock);
+        }
+
+        // Return freshly filled list:
+        self.emit(Instruction::ReturnValue);
+
+        // Fetch code for listcomp function:
+        let code = self.pop_code_object();
+
+        // List comprehension code:
+        self.emit(Instruction::LoadConst {
+            value: bytecode::Constant::Code {
+                code: Box::new(code),
+            },
+        });
+
+        // List comprehension function name:
+        self.emit(Instruction::LoadConst {
+            value: bytecode::Constant::String { value: name },
+        });
+
+        // Turn code object into function object:
+        self.emit(Instruction::MakeFunction {
+            flags: bytecode::FunctionOpArg::empty(),
+        });
+
+        // Evaluate iterated item:
+        self.compile_expression(&generators[0].iter)?;
+
+        // Get iterator / turn item into an iterator
+        self.emit(Instruction::GetIter);
+
+        // Call just created <listcomp> function:
+        self.emit(Instruction::CallFunction {
+            typ: CallType::Positional(1),
+        });
+        Ok(())
+    }
+
+    fn compile_string(&mut self, string: &ast::StringGroup) -> Result<(), CompileError> {
+        match string {
+            ast::StringGroup::Joined { values } => {
+                for value in values {
+                    self.compile_string(value)?;
+                }
+                self.emit(Instruction::BuildString { size: values.len() })
+            }
+            ast::StringGroup::Constant { value } => {
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::String {
+                        value: value.to_string(),
+                    },
+                });
+            }
+            ast::StringGroup::FormattedValue {
+                value,
+                conversion,
+                spec,
+            } => {
+                self.compile_expression(value)?;
+                self.emit(Instruction::FormatValue {
+                    conversion: *conversion,
+                    spec: spec.clone(),
+                });
+            }
+        }
+        Ok(())
+    }
+
+    // Scope helpers:
+    fn enter_scope(&mut self) {
+        // println!("Enter scope {:?}", self.scope_stack);
+        // Enter first subscope!
+        let scope = self.scope_stack.last_mut().unwrap().sub_scopes.remove(0);
+        self.scope_stack.push(scope);
+    }
+
+    fn leave_scope(&mut self) {
+        // println!("Leave scope {:?}", self.scope_stack);
+        let scope = self.scope_stack.pop().unwrap();
+        assert!(scope.sub_scopes.is_empty());
+    }
+
+    fn lookup_name(&self, name: &str) -> &SymbolRole {
+        // println!("Looking up {:?}", name);
+        let scope = self.scope_stack.last().unwrap();
+        scope.lookup(name).unwrap()
+    }
+
+    // Low level helper functions:
+    fn emit(&mut self, instruction: Instruction) {
+        let location = self.current_source_location.clone();
+        let cur_code_obj = self.current_code_object();
+        cur_code_obj.instructions.push(instruction);
+        cur_code_obj.locations.push(location);
+        // TODO: insert source filename
+    }
+
+    fn current_code_object(&mut self) -> &mut CodeObject {
+        self.code_object_stack.last_mut().unwrap()
+    }
+
+    // Generate a new label
+    fn new_label(&mut self) -> Label {
+        let l = self.nxt_label;
+        self.nxt_label += 1;
+        l
+    }
+
+    // Assign current position the given label
+    fn set_label(&mut self, label: Label) {
+        let position = self.current_code_object().instructions.len();
+        // assert!(label not in self.label_map)
+        self.current_code_object().label_map.insert(label, position);
+    }
+
+    fn set_source_location(&mut self, location: &ast::Location) {
+        self.current_source_location = location.clone();
+    }
+
+    fn get_source_line_number(&mut self) -> usize {
+        self.current_source_location.get_row()
+    }
+
+    fn create_qualified_name(&self, name: &str, suffix: &str) -> String {
+        if let Some(ref qualified_path) = self.current_qualified_path {
+            format!("{}.{}{}", qualified_path, name, suffix)
+        } else {
+            format!("{}{}", name, suffix)
+        }
+    }
+
+    fn mark_generator(&mut self) {
+        self.current_code_object().is_generator = true;
+    }
+}
+
+fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option<String>) {
+    if let Some(val) = body.get(0) {
+        if let ast::Statement::Expression { ref expression } = val.node {
+            if let ast::Expression::String { ref value } = expression {
+                if let ast::StringGroup::Constant { ref value } = value {
+                    if let Some((_, body_rest)) = body.split_first() {
+                        return (body_rest, Some(value.to_string()));
+                    }
+                }
+            }
+        }
+    }
+    (body, None)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Compiler;
+    use crate::bytecode::CodeObject;
+    use crate::bytecode::Constant::*;
+    use crate::bytecode::Instruction::*;
+    use crate::symboltable::make_symbol_table;
+    use rustpython_parser::parser;
+
+    fn compile_exec(source: &str) -> CodeObject {
+        let mut compiler = Compiler::new();
+        compiler.source_path = Some("source_path".to_string());
+        compiler.push_new_code_object("<module>".to_string());
+        let ast = parser::parse_program(&source.to_string()).unwrap();
+        let symbol_scope = make_symbol_table(&ast).unwrap();
+        compiler.compile_program(&ast, symbol_scope).unwrap();
+        compiler.pop_code_object()
+    }
+
+    #[test]
+    fn test_if_ors() {
+        let code = compile_exec("if True or False or False:\n pass\n");
+        assert_eq!(
+            vec![
+                LoadConst {
+                    value: Boolean { value: true }
+                },
+                JumpIf { target: 1 },
+                LoadConst {
+                    value: Boolean { value: false }
+                },
+                JumpIf { target: 1 },
+                LoadConst {
+                    value: Boolean { value: false }
+                },
+                JumpIfFalse { target: 0 },
+                Pass,
+                LoadConst { value: None },
+                ReturnValue
+            ],
+            code.instructions
+        );
+    }
+
+    #[test]
+    fn test_if_ands() {
+        let code = compile_exec("if True and False and False:\n pass\n");
+        assert_eq!(
+            vec![
+                LoadConst {
+                    value: Boolean { value: true }
+                },
+                JumpIfFalse { target: 0 },
+                LoadConst {
+                    value: Boolean { value: false }
+                },
+                JumpIfFalse { target: 0 },
+                LoadConst {
+                    value: Boolean { value: false }
+                },
+                JumpIfFalse { target: 0 },
+                Pass,
+                LoadConst { value: None },
+                ReturnValue
+            ],
+            code.instructions
+        );
+    }
+
+    #[test]
+    fn test_if_mixed() {
+        let code = compile_exec("if (True and False) or (False and True):\n pass\n");
+        assert_eq!(
+            vec![
+                LoadConst {
+                    value: Boolean { value: true }
+                },
+                JumpIfFalse { target: 2 },
+                LoadConst {
+                    value: Boolean { value: false }
+                },
+                JumpIf { target: 1 },
+                LoadConst {
+                    value: Boolean { value: false }
+                },
+                JumpIfFalse { target: 0 },
+                LoadConst {
+                    value: Boolean { value: true }
+                },
+                JumpIfFalse { target: 0 },
+                Pass,
+                LoadConst { value: None },
+                ReturnValue
+            ],
+            code.instructions
+        );
+    }
+}
diff --git a/src/error.rs b/src/error.rs
new file mode 100644
index 00000000..af763168
--- /dev/null
+++ b/src/error.rs
@@ -0,0 +1,67 @@
+use rustpython_parser::error::ParseError;
+use rustpython_parser::lexer::Location;
+
+use std::error::Error;
+use std::fmt;
+
+#[derive(Debug)]
+pub struct CompileError {
+    pub error: CompileErrorType,
+    pub location: Location,
+}
+
+impl From<ParseError> for CompileError {
+    fn from(error: ParseError) -> Self {
+        CompileError {
+            error: CompileErrorType::Parse(error),
+            location: Default::default(), // TODO: extract location from parse error!
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum CompileErrorType {
+    /// Invalid assignment, cannot store value in target.
+    Assign(&'static str),
+    /// Invalid delete
+    Delete(&'static str),
+    /// Expected an expression got a statement
+    ExpectExpr,
+    /// Parser error
+    Parse(ParseError),
+    SyntaxError(String),
+    /// Multiple `*` detected
+    StarArgs,
+    /// Break statement outside of loop.
+    InvalidBreak,
+    /// Continue statement outside of loop.
+    InvalidContinue,
+    InvalidReturn,
+    InvalidYield,
+}
+
+impl fmt::Display for CompileError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match &self.error {
+            CompileErrorType::Assign(target) => write!(f, "can't assign to {}", target),
+            CompileErrorType::Delete(target) => write!(f, "can't delete {}", target),
+            CompileErrorType::ExpectExpr => write!(f, "Expecting expression, got statement"),
+            CompileErrorType::Parse(err) => write!(f, "{}", err),
+            CompileErrorType::SyntaxError(err) => write!(f, "{}", err),
+            CompileErrorType::StarArgs => write!(f, "Two starred expressions in assignment"),
+            CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"),
+            CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"),
+            CompileErrorType::InvalidReturn => write!(f, "'return' outside function"),
+            CompileErrorType::InvalidYield => write!(f, "'yield' outside function"),
+        }?;
+
+        // Print line number:
+        write!(f, " at line {:?}", self.location.get_row())
+    }
+}
+
+impl Error for CompileError {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        None
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
new file mode 100644
index 00000000..1f0805c6
--- /dev/null
+++ b/src/lib.rs
@@ -0,0 +1,7 @@
+#[macro_use]
+extern crate log;
+
+pub mod bytecode;
+pub mod compile;
+pub mod error;
+mod symboltable;
diff --git a/src/symboltable.rs b/src/symboltable.rs
new file mode 100644
index 00000000..9687b0ec
--- /dev/null
+++ b/src/symboltable.rs
@@ -0,0 +1,588 @@
+/* Python code is pre-scanned for symbols in the ast.
+
+This ensures that global and nonlocal keywords are picked up.
+Then the compiler can use the symbol table to generate proper
+load and store instructions for names.
+
+Inspirational file: https://github.com/python/cpython/blob/master/Python/symtable.c
+*/
+
+use crate::error::{CompileError, CompileErrorType};
+use rustpython_parser::ast;
+use rustpython_parser::lexer::Location;
+use std::collections::HashMap;
+
+pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolScope, SymbolTableError> {
+    let mut builder = SymbolTableBuilder::new();
+    builder.enter_scope();
+    builder.scan_program(program)?;
+    assert_eq!(builder.scopes.len(), 1);
+
+    let symbol_table = builder.scopes.pop().unwrap();
+    analyze_symbol_table(&symbol_table, None)?;
+    Ok(symbol_table)
+}
+
+pub fn statements_to_symbol_table(
+    statements: &[ast::LocatedStatement],
+) -> Result<SymbolScope, SymbolTableError> {
+    let mut builder = SymbolTableBuilder::new();
+    builder.enter_scope();
+    builder.scan_statements(statements)?;
+    assert_eq!(builder.scopes.len(), 1);
+
+    let symbol_table = builder.scopes.pop().unwrap();
+    analyze_symbol_table(&symbol_table, None)?;
+    Ok(symbol_table)
+}
+
+#[derive(Debug)]
+pub enum SymbolRole {
+    Global,
+    Nonlocal,
+    Used,
+    Assigned,
+}
+
+/// Captures all symbols in the current scope, and has a list of subscopes in this scope.
+pub struct SymbolScope {
+    /// A set of symbols present on this scope level.
+    pub symbols: HashMap<String, SymbolRole>,
+
+    /// A list of subscopes in the order as found in the
+    /// AST nodes.
+    pub sub_scopes: Vec<SymbolScope>,
+}
+
+#[derive(Debug)]
+pub struct SymbolTableError {
+    error: String,
+    location: Location,
+}
+
+impl From<SymbolTableError> for CompileError {
+    fn from(error: SymbolTableError) -> Self {
+        CompileError {
+            error: CompileErrorType::SyntaxError(error.error),
+            location: error.location,
+        }
+    }
+}
+
+type SymbolTableResult = Result<(), SymbolTableError>;
+
+impl SymbolScope {
+    pub fn new() -> Self {
+        SymbolScope {
+            symbols: HashMap::new(),
+            sub_scopes: vec![],
+        }
+    }
+
+    pub fn lookup(&self, name: &str) -> Option<&SymbolRole> {
+        self.symbols.get(name)
+    }
+}
+
+impl std::fmt::Debug for SymbolScope {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            "SymbolScope({:?} symbols, {:?} sub scopes)",
+            self.symbols.len(),
+            self.sub_scopes.len()
+        )
+    }
+}
+
+/* Perform some sort of analysis on nonlocals, globals etc..
+  See also: https://github.com/python/cpython/blob/master/Python/symtable.c#L410
+*/
+fn analyze_symbol_table(
+    symbol_scope: &SymbolScope,
+    parent_symbol_scope: Option<&SymbolScope>,
+) -> SymbolTableResult {
+    // Analyze sub scopes:
+    for sub_scope in &symbol_scope.sub_scopes {
+        analyze_symbol_table(&sub_scope, Some(symbol_scope))?;
+    }
+
+    // Analyze symbols:
+    for (symbol_name, symbol_role) in &symbol_scope.symbols {
+        analyze_symbol(symbol_name, symbol_role, parent_symbol_scope)?;
+    }
+
+    Ok(())
+}
+
+#[allow(clippy::single_match)]
+fn analyze_symbol(
+    symbol_name: &str,
+    symbol_role: &SymbolRole,
+    parent_symbol_scope: Option<&SymbolScope>,
+) -> SymbolTableResult {
+    match symbol_role {
+        SymbolRole::Nonlocal => {
+            // check if name is defined in parent scope!
+            if let Some(parent_symbol_scope) = parent_symbol_scope {
+                if !parent_symbol_scope.symbols.contains_key(symbol_name) {
+                    return Err(SymbolTableError {
+                        error: format!("no binding for nonlocal '{}' found", symbol_name),
+                        location: Default::default(),
+                    });
+                }
+            } else {
+                return Err(SymbolTableError {
+                    error: format!(
+                        "nonlocal {} defined at place without an enclosing scope",
+                        symbol_name
+                    ),
+                    location: Default::default(),
+                });
+            }
+        }
+        // TODO: add more checks for globals
+        _ => {}
+    }
+    Ok(())
+}
+
+pub struct SymbolTableBuilder {
+    // Scope stack.
+    pub scopes: Vec<SymbolScope>,
+}
+
+impl SymbolTableBuilder {
+    pub fn new() -> Self {
+        SymbolTableBuilder { scopes: vec![] }
+    }
+
+    pub fn enter_scope(&mut self) {
+        let scope = SymbolScope::new();
+        self.scopes.push(scope);
+    }
+
+    fn leave_scope(&mut self) {
+        // Pop scope and add to subscopes of parent scope.
+        let scope = self.scopes.pop().unwrap();
+        self.scopes.last_mut().unwrap().sub_scopes.push(scope);
+    }
+
+    pub fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult {
+        self.scan_statements(&program.statements)?;
+        Ok(())
+    }
+
+    pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) -> SymbolTableResult {
+        for statement in statements {
+            self.scan_statement(statement)?;
+        }
+        Ok(())
+    }
+
+    fn scan_parameters(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
+        for parameter in parameters {
+            self.scan_parameter(parameter)?;
+        }
+        Ok(())
+    }
+
+    fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
+        self.register_name(&parameter.arg, SymbolRole::Assigned)
+    }
+
+    fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
+        for parameter in parameters {
+            self.scan_parameter_annotation(parameter)?;
+        }
+        Ok(())
+    }
+
+    fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
+        if let Some(annotation) = &parameter.annotation {
+            self.scan_expression(&annotation)?;
+        }
+        Ok(())
+    }
+
+    fn scan_statement(&mut self, statement: &ast::LocatedStatement) -> SymbolTableResult {
+        match &statement.node {
+            ast::Statement::Global { names } => {
+                for name in names {
+                    self.register_name(name, SymbolRole::Global)?;
+                }
+            }
+            ast::Statement::Nonlocal { names } => {
+                for name in names {
+                    self.register_name(name, SymbolRole::Nonlocal)?;
+                }
+            }
+            ast::Statement::FunctionDef {
+                name,
+                body,
+                args,
+                decorator_list,
+                returns,
+            }
+            | ast::Statement::AsyncFunctionDef {
+                name,
+                body,
+                args,
+                decorator_list,
+                returns,
+            } => {
+                self.scan_expressions(decorator_list)?;
+                self.register_name(name, SymbolRole::Assigned)?;
+
+                self.enter_function(args)?;
+
+                self.scan_statements(body)?;
+                if let Some(expression) = returns {
+                    self.scan_expression(expression)?;
+                }
+                self.leave_scope();
+            }
+            ast::Statement::ClassDef {
+                name,
+                body,
+                bases,
+                keywords,
+                decorator_list,
+            } => {
+                self.register_name(name, SymbolRole::Assigned)?;
+                self.enter_scope();
+                self.scan_statements(body)?;
+                self.leave_scope();
+                self.scan_expressions(bases)?;
+                for keyword in keywords {
+                    self.scan_expression(&keyword.value)?;
+                }
+                self.scan_expressions(decorator_list)?;
+            }
+            ast::Statement::Expression { expression } => self.scan_expression(expression)?,
+            ast::Statement::If { test, body, orelse } => {
+                self.scan_expression(test)?;
+                self.scan_statements(body)?;
+                if let Some(code) = orelse {
+                    self.scan_statements(code)?;
+                }
+            }
+            ast::Statement::For {
+                target,
+                iter,
+                body,
+                orelse,
+            }
+            | ast::Statement::AsyncFor {
+                target,
+                iter,
+                body,
+                orelse,
+            } => {
+                self.scan_expression(target)?;
+                self.scan_expression(iter)?;
+                self.scan_statements(body)?;
+                if let Some(code) = orelse {
+                    self.scan_statements(code)?;
+                }
+            }
+            ast::Statement::While { test, body, orelse } => {
+                self.scan_expression(test)?;
+                self.scan_statements(body)?;
+                if let Some(code) = orelse {
+                    self.scan_statements(code)?;
+                }
+            }
+            ast::Statement::Break | ast::Statement::Continue | ast::Statement::Pass => {
+                // No symbols here.
+            }
+            ast::Statement::Import { import_parts } => {
+                for part in import_parts {
+                    if let Some(alias) = &part.alias {
+                        // `import mymodule as myalias`
+                        // `from mymodule import myimportname as myalias`
+                        self.register_name(alias, SymbolRole::Assigned)?;
+                    } else if let Some(symbol) = &part.symbol {
+                        // `from mymodule import myimport`
+                        self.register_name(symbol, SymbolRole::Assigned)?;
+                    } else {
+                        // `import module`
+                        self.register_name(&part.module, SymbolRole::Assigned)?;
+                    }
+                }
+            }
+            ast::Statement::Return { value } => {
+                if let Some(expression) = value {
+                    self.scan_expression(expression)?;
+                }
+            }
+            ast::Statement::Assert { test, msg } => {
+                self.scan_expression(test)?;
+                if let Some(expression) = msg {
+                    self.scan_expression(expression)?;
+                }
+            }
+            ast::Statement::Delete { targets } => {
+                self.scan_expressions(targets)?;
+            }
+            ast::Statement::Assign { targets, value } => {
+                self.scan_expressions(targets)?;
+                self.scan_expression(value)?;
+            }
+            ast::Statement::AugAssign { target, value, .. } => {
+                self.scan_expression(target)?;
+                self.scan_expression(value)?;
+            }
+            ast::Statement::With { items, body } => {
+                for item in items {
+                    self.scan_expression(&item.context_expr)?;
+                    if let Some(expression) = &item.optional_vars {
+                        self.scan_expression(expression)?;
+                    }
+                }
+                self.scan_statements(body)?;
+            }
+            ast::Statement::Try {
+                body,
+                handlers,
+                orelse,
+                finalbody,
+            } => {
+                self.scan_statements(body)?;
+                for handler in handlers {
+                    if let Some(expression) = &handler.typ {
+                        self.scan_expression(expression)?;
+                    }
+                    if let Some(name) = &handler.name {
+                        self.register_name(name, SymbolRole::Assigned)?;
+                    }
+                    self.scan_statements(&handler.body)?;
+                }
+                if let Some(code) = orelse {
+                    self.scan_statements(code)?;
+                }
+                if let Some(code) = finalbody {
+                    self.scan_statements(code)?;
+                }
+            }
+            ast::Statement::Raise { exception, cause } => {
+                if let Some(expression) = exception {
+                    self.scan_expression(expression)?;
+                }
+                if let Some(expression) = cause {
+                    self.scan_expression(expression)?;
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn scan_expressions(&mut self, expressions: &[ast::Expression]) -> SymbolTableResult {
+        for expression in expressions {
+            self.scan_expression(expression)?;
+        }
+        Ok(())
+    }
+
+    fn scan_expression(&mut self, expression: &ast::Expression) -> SymbolTableResult {
+        match expression {
+            ast::Expression::Binop { a, b, .. } => {
+                self.scan_expression(a)?;
+                self.scan_expression(b)?;
+            }
+            ast::Expression::BoolOp { a, b, .. } => {
+                self.scan_expression(a)?;
+                self.scan_expression(b)?;
+            }
+            ast::Expression::Compare { vals, .. } => {
+                self.scan_expressions(vals)?;
+            }
+            ast::Expression::Subscript { a, b } => {
+                self.scan_expression(a)?;
+                self.scan_expression(b)?;
+            }
+            ast::Expression::Attribute { value, .. } => {
+                self.scan_expression(value)?;
+            }
+            ast::Expression::Dict { elements } => {
+                for (key, value) in elements {
+                    if let Some(key) = key {
+                        self.scan_expression(key)?;
+                    } else {
+                        // dict unpacking marker
+                    }
+                    self.scan_expression(value)?;
+                }
+            }
+            ast::Expression::Await { value } => {
+                self.scan_expression(value)?;
+            }
+            ast::Expression::Yield { value } => {
+                if let Some(expression) = value {
+                    self.scan_expression(expression)?;
+                }
+            }
+            ast::Expression::YieldFrom { value } => {
+                self.scan_expression(value)?;
+            }
+            ast::Expression::Unop { a, .. } => {
+                self.scan_expression(a)?;
+            }
+            ast::Expression::True
+            | ast::Expression::False
+            | ast::Expression::None
+            | ast::Expression::Ellipsis => {}
+            ast::Expression::Number { .. } => {}
+            ast::Expression::Starred { value } => {
+                self.scan_expression(value)?;
+            }
+            ast::Expression::Bytes { .. } => {}
+            ast::Expression::Tuple { elements }
+            | ast::Expression::Set { elements }
+            | ast::Expression::List { elements }
+            | ast::Expression::Slice { elements } => {
+                self.scan_expressions(elements)?;
+            }
+            ast::Expression::Comprehension { kind, generators } => {
+                match **kind {
+                    ast::ComprehensionKind::GeneratorExpression { ref element }
+                    | ast::ComprehensionKind::List { ref element }
+                    | ast::ComprehensionKind::Set { ref element } => {
+                        self.scan_expression(element)?;
+                    }
+                    ast::ComprehensionKind::Dict { ref key, ref value } => {
+                        self.scan_expression(&key)?;
+                        self.scan_expression(&value)?;
+                    }
+                }
+
+                for generator in generators {
+                    self.scan_expression(&generator.target)?;
+                    self.scan_expression(&generator.iter)?;
+                    for if_expr in &generator.ifs {
+                        self.scan_expression(if_expr)?;
+                    }
+                }
+            }
+            ast::Expression::Call {
+                function,
+                args,
+                keywords,
+            } => {
+                self.scan_expression(function)?;
+                self.scan_expressions(args)?;
+                for keyword in keywords {
+                    self.scan_expression(&keyword.value)?;
+                }
+            }
+            ast::Expression::String { value } => {
+                self.scan_string_group(value)?;
+            }
+            ast::Expression::Identifier { name } => {
+                self.register_name(name, SymbolRole::Used)?;
+            }
+            ast::Expression::Lambda { args, body } => {
+                self.enter_function(args)?;
+                self.scan_expression(body)?;
+                self.leave_scope();
+            }
+            ast::Expression::IfExpression { test, body, orelse } => {
+                self.scan_expression(test)?;
+                self.scan_expression(body)?;
+                self.scan_expression(orelse)?;
+            }
+        }
+        Ok(())
+    }
+
+    fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult {
+        // Evaluate eventual default parameters:
+        self.scan_expressions(&args.defaults)?;
+        for kw_default in &args.kw_defaults {
+            if let Some(expression) = kw_default {
+                self.scan_expression(&expression)?;
+            }
+        }
+
+        // Annotations are scanned in outer scope:
+        self.scan_parameters_annotations(&args.args)?;
+        self.scan_parameters_annotations(&args.kwonlyargs)?;
+        if let ast::Varargs::Named(name) = &args.vararg {
+            self.scan_parameter_annotation(name)?;
+        }
+        if let ast::Varargs::Named(name) = &args.kwarg {
+            self.scan_parameter_annotation(name)?;
+        }
+
+        self.enter_scope();
+
+        // Fill scope with parameter names:
+        self.scan_parameters(&args.args)?;
+        self.scan_parameters(&args.kwonlyargs)?;
+        if let ast::Varargs::Named(name) = &args.vararg {
+            self.scan_parameter(name)?;
+        }
+        if let ast::Varargs::Named(name) = &args.kwarg {
+            self.scan_parameter(name)?;
+        }
+        Ok(())
+    }
+
+    fn scan_string_group(&mut self, group: &ast::StringGroup) -> SymbolTableResult {
+        match group {
+            ast::StringGroup::Constant { .. } => {}
+            ast::StringGroup::FormattedValue { value, .. } => {
+                self.scan_expression(value)?;
+            }
+            ast::StringGroup::Joined { values } => {
+                for subgroup in values {
+                    self.scan_string_group(subgroup)?;
+                }
+            }
+        }
+        Ok(())
+    }
+
+    #[allow(clippy::single_match)]
+    fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult {
+        let scope_depth = self.scopes.len();
+        let current_scope = self.scopes.last_mut().unwrap();
+        let location = Default::default();
+        if current_scope.symbols.contains_key(name) {
+            // Role already set..
+            match role {
+                SymbolRole::Global => {
+                    return Err(SymbolTableError {
+                        error: format!("name '{}' is used prior to global declaration", name),
+                        location,
+                    })
+                }
+                SymbolRole::Nonlocal => {
+                    return Err(SymbolTableError {
+                        error: format!("name '{}' is used prior to nonlocal declaration", name),
+                        location,
+                    })
+                }
+                _ => {
+                    // Ok?
+                }
+            }
+        } else {
+            match role {
+                SymbolRole::Nonlocal => {
+                    if scope_depth < 2 {
+                        return Err(SymbolTableError {
+                            error: format!("cannot define nonlocal '{}' at top level.", name),
+                            location,
+                        });
+                    }
+                }
+                _ => {
+                    // Ok!
+                }
+            }
+            current_scope.symbols.insert(name.to_string(), role);
+        }
+        Ok(())
+    }
+}

From e85f8b18df184ca8fdec95edee6d48d4ca75ca60 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 12 Jun 2019 22:04:59 -0500
Subject: [PATCH 002/459] Add doc comments and individual compile functions

---
 src/bytecode.rs |  4 ----
 src/compile.rs  | 55 +++++++++++++++++++++++++++++++++++++++----------
 src/lib.rs      |  3 +++
 3 files changed, 47 insertions(+), 15 deletions(-)

diff --git a/src/bytecode.rs b/src/bytecode.rs
index 0fc226b5..aede69c9 100644
--- a/src/bytecode.rs
+++ b/src/bytecode.rs
@@ -1,10 +1,6 @@
 //! Implement python as a virtual machine with bytecodes. This module
 //! implements bytecode structure.
 
-/*
- * Primitive instruction type, which can be encoded and decoded.
- */
-
 use bitflags::bitflags;
 use num_bigint::BigInt;
 use num_complex::Complex64;
diff --git a/src/compile.rs b/src/compile.rs
index dec158e9..30510c20 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -24,33 +24,66 @@ struct Compiler {
 
 /// Compile a given sourcecode into a bytecode object.
 pub fn compile(source: &str, mode: &Mode, source_path: String) -> Result<CodeObject, CompileError> {
-    let mut compiler = Compiler::new();
-    compiler.source_path = Some(source_path);
-    compiler.push_new_code_object("<module>".to_string());
-
     match mode {
         Mode::Exec => {
             let ast = parser::parse_program(source)?;
-            let symbol_table = make_symbol_table(&ast)?;
-            compiler.compile_program(&ast, symbol_table)
+            compile_program(ast, source_path)
         }
         Mode::Eval => {
             let statement = parser::parse_statement(source)?;
-            let symbol_table = statements_to_symbol_table(&statement)?;
-            compiler.compile_statement_eval(&statement, symbol_table)
+            compile_statement_eval(statement, source_path)
         }
         Mode::Single => {
             let ast = parser::parse_program(source)?;
-            let symbol_table = make_symbol_table(&ast)?;
-            compiler.compile_program_single(&ast, symbol_table)
+            compile_program_single(ast, source_path)
         }
-    }?;
+    }
+}
 
+/// A helper function for the shared code of the different compile functions
+fn with_compiler(
+    source_path: String,
+    f: impl FnOnce(&mut Compiler) -> Result<(), CompileError>,
+) -> Result<CodeObject, CompileError> {
+    let mut compiler = Compiler::new();
+    compiler.source_path = Some(source_path);
+    compiler.push_new_code_object("<module>".to_string());
+    f(&mut compiler)?;
     let code = compiler.pop_code_object();
     trace!("Compilation completed: {:?}", code);
     Ok(code)
 }
 
+/// Compile a standard Python program to bytecode
+pub fn compile_program(ast: ast::Program, source_path: String) -> Result<CodeObject, CompileError> {
+    with_compiler(source_path, |compiler| {
+        let symbol_table = make_symbol_table(&ast)?;
+        compiler.compile_program(&ast, symbol_table)
+    })
+}
+
+/// Compile a single Python expression to bytecode
+pub fn compile_statement_eval(
+    statement: Vec<ast::LocatedStatement>,
+    source_path: String,
+) -> Result<CodeObject, CompileError> {
+    with_compiler(source_path, |compiler| {
+        let symbol_table = statements_to_symbol_table(&statement)?;
+        compiler.compile_statement_eval(&statement, symbol_table)
+    })
+}
+
+/// Compile a Python program to bytecode for the context of a REPL
+pub fn compile_program_single(
+    ast: ast::Program,
+    source_path: String,
+) -> Result<CodeObject, CompileError> {
+    with_compiler(source_path, |compiler| {
+        let symbol_table = make_symbol_table(&ast)?;
+        compiler.compile_program_single(&ast, symbol_table)
+    })
+}
+
 pub enum Mode {
     Exec,
     Eval,
diff --git a/src/lib.rs b/src/lib.rs
index 1f0805c6..382647d1 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,3 +1,6 @@
+//! Compile a Python AST or source code into bytecode consumable by RustPython or
+//! (eventually) CPython.
+
 #[macro_use]
 extern crate log;
 

From a932b729d82ea22a25d28ef415ecfc3a8c178519 Mon Sep 17 00:00:00 2001
From: Aviv Palivoda <palaviv@gmail.com>
Date: Thu, 13 Jun 2019 20:41:37 +0300
Subject: [PATCH 003/459] Import all from_list in one __import__ call

---
 src/bytecode.rs    |  4 ++--
 src/compile.rs     | 56 ++++++++++++++++++++++++++++++++--------------
 src/symboltable.rs | 20 ++++++++++++-----
 3 files changed, 55 insertions(+), 25 deletions(-)

diff --git a/src/bytecode.rs b/src/bytecode.rs
index aede69c9..c87bc0ae 100644
--- a/src/bytecode.rs
+++ b/src/bytecode.rs
@@ -50,7 +50,7 @@ pub enum NameScope {
 pub enum Instruction {
     Import {
         name: String,
-        symbol: Option<String>,
+        symbols: Vec<String>,
     },
     ImportStar {
         name: String,
@@ -330,7 +330,7 @@ impl Instruction {
         }
 
         match self {
-            Import { name, symbol } => w!(Import, name, format!("{:?}", symbol)),
+            Import { name, symbols } => w!(Import, name, format!("{:?}", symbols)),
             ImportStar { name } => w!(ImportStar, name),
             LoadName { name, scope } => w!(LoadName, name, format!("{:?}", scope)),
             StoreName { name, scope } => w!(StoreName, name, format!("{:?}", scope)),
diff --git a/src/compile.rs b/src/compile.rs
index 30510c20..dc83fad2 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -250,29 +250,51 @@ impl Compiler {
             ast::Statement::Import { import_parts } => {
                 for ast::SingleImport {
                     module,
-                    symbol,
+                    symbols,
                     alias,
                 } in import_parts
                 {
-                    match symbol {
-                        Some(name) if name == "*" => {
-                            self.emit(Instruction::ImportStar {
-                                name: module.clone(),
-                            });
-                        }
-                        _ => {
+                    if let Some(alias) = alias {
+                        self.emit(Instruction::Import {
+                            name: module.clone(),
+                            symbols: vec![],
+                        });
+                        self.store_name(&alias);
+                    } else {
+                        if symbols.is_empty() {
                             self.emit(Instruction::Import {
                                 name: module.clone(),
-                                symbol: symbol.clone(),
+                                symbols: vec![],
                             });
-                            let name = match alias {
-                                Some(alias) => alias.clone(),
-                                None => match symbol {
-                                    Some(symbol) => symbol.clone(),
-                                    None => module.clone(),
-                                },
-                            };
-                            self.store_name(&name);
+                            self.store_name(&module.clone());
+                        } else {
+                            let mut import_star = false;
+                            let mut symbols_strings = vec![];
+                            let mut names = vec![];
+                            for ast::ImportSymbol { symbol, alias } in symbols {
+                                if symbol == "*" {
+                                    import_star = true;
+                                }
+                                symbols_strings.push(symbol.to_string());
+                                names.insert(
+                                    0,
+                                    match alias {
+                                        Some(alias) => alias,
+                                        None => symbol,
+                                    },
+                                );
+                            }
+                            if import_star {
+                                self.emit(Instruction::ImportStar {
+                                    name: module.clone(),
+                                });
+                            } else {
+                                self.emit(Instruction::Import {
+                                    name: module.clone(),
+                                    symbols: symbols_strings,
+                                });
+                                names.iter().for_each(|name| self.store_name(&name));
+                            }
                         }
                     }
                 }
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 9687b0ec..04e5d9ed 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -300,14 +300,22 @@ impl SymbolTableBuilder {
                 for part in import_parts {
                     if let Some(alias) = &part.alias {
                         // `import mymodule as myalias`
-                        // `from mymodule import myimportname as myalias`
                         self.register_name(alias, SymbolRole::Assigned)?;
-                    } else if let Some(symbol) = &part.symbol {
-                        // `from mymodule import myimport`
-                        self.register_name(symbol, SymbolRole::Assigned)?;
                     } else {
-                        // `import module`
-                        self.register_name(&part.module, SymbolRole::Assigned)?;
+                        if part.symbols.is_empty() {
+                            // `import module`
+                            self.register_name(&part.module, SymbolRole::Assigned)?;
+                        } else {
+                            // `from mymodule import myimport`
+                            for symbol in &part.symbols {
+                                if let Some(alias) = &symbol.alias {
+                                    // `from mymodule import myimportname as myalias`
+                                    self.register_name(alias, SymbolRole::Assigned)?;
+                                } else {
+                                    self.register_name(&symbol.symbol, SymbolRole::Assigned)?;
+                                }
+                            }
+                        }
                     }
                 }
             }

From 3a1dd34987fdf37b701caa16b7566177d368313f Mon Sep 17 00:00:00 2001
From: Aviv Palivoda <palaviv@gmail.com>
Date: Fri, 14 Jun 2019 08:45:30 +0300
Subject: [PATCH 004/459] Improve compiler import

---
 src/compile.rs | 61 +++++++++++++++++++++++++-------------------------
 1 file changed, 30 insertions(+), 31 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index dc83fad2..8b806b1d 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -255,46 +255,45 @@ impl Compiler {
                 } in import_parts
                 {
                     if let Some(alias) = alias {
+                        // import module as alias
                         self.emit(Instruction::Import {
                             name: module.clone(),
                             symbols: vec![],
                         });
                         self.store_name(&alias);
+                    } else if symbols.is_empty() {
+                        // import module
+                        self.emit(Instruction::Import {
+                            name: module.clone(),
+                            symbols: vec![],
+                        });
+                        self.store_name(&module.clone());
                     } else {
-                        if symbols.is_empty() {
-                            self.emit(Instruction::Import {
+                        let import_star = symbols
+                            .iter()
+                            .any(|import_symbol| import_symbol.symbol == "*");
+                        if import_star {
+                            // from module import *
+                            self.emit(Instruction::ImportStar {
                                 name: module.clone(),
-                                symbols: vec![],
                             });
-                            self.store_name(&module.clone());
                         } else {
-                            let mut import_star = false;
-                            let mut symbols_strings = vec![];
-                            let mut names = vec![];
-                            for ast::ImportSymbol { symbol, alias } in symbols {
-                                if symbol == "*" {
-                                    import_star = true;
-                                }
-                                symbols_strings.push(symbol.to_string());
-                                names.insert(
-                                    0,
-                                    match alias {
-                                        Some(alias) => alias,
-                                        None => symbol,
-                                    },
-                                );
-                            }
-                            if import_star {
-                                self.emit(Instruction::ImportStar {
-                                    name: module.clone(),
-                                });
-                            } else {
-                                self.emit(Instruction::Import {
-                                    name: module.clone(),
-                                    symbols: symbols_strings,
-                                });
-                                names.iter().for_each(|name| self.store_name(&name));
-                            }
+                            // from module import symbol
+                            // from module import symbol as alias
+                            let (names, symbols_strings): (Vec<String>, Vec<String>) = symbols
+                                .iter()
+                                .map(|ast::ImportSymbol { symbol, alias }| {
+                                    (
+                                        alias.clone().unwrap_or_else(|| symbol.to_string()),
+                                        symbol.to_string(),
+                                    )
+                                })
+                                .unzip();
+                            self.emit(Instruction::Import {
+                                name: module.clone(),
+                                symbols: symbols_strings,
+                            });
+                            names.iter().rev().for_each(|name| self.store_name(&name));
                         }
                     }
                 }

From 048ecf48714903959aa1561597ce3aa653905c81 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 14 Jun 2019 10:42:45 -0500
Subject: [PATCH 005/459] Add py_compile_bytecode! macro

---
 src/bytecode.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/bytecode.rs b/src/bytecode.rs
index aede69c9..6fdfaf32 100644
--- a/src/bytecode.rs
+++ b/src/bytecode.rs
@@ -9,6 +9,8 @@ use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
 use std::fmt;
 
+pub use ast::{ConversionFlag, Location};
+
 /// Primary container of a single code object. Each python function has
 /// a codeobject. Also a module has a codeobject.
 #[derive(Clone, PartialEq, Serialize, Deserialize)]

From 71b3fca2eda3c1a035474dc5b27e64232f8fc4a6 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 14 Jun 2019 10:47:28 -0500
Subject: [PATCH 006/459] Change imp_get_frozen_object

---
 src/bytecode.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/bytecode.rs b/src/bytecode.rs
index 6fdfaf32..aede69c9 100644
--- a/src/bytecode.rs
+++ b/src/bytecode.rs
@@ -9,8 +9,6 @@ use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
 use std::fmt;
 
-pub use ast::{ConversionFlag, Location};
-
 /// Primary container of a single code object. Each python function has
 /// a codeobject. Also a module has a codeobject.
 #[derive(Clone, PartialEq, Serialize, Deserialize)]

From d7b4837e26904ef80d10b21fb00dc1f4faaa265a Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 15 Jun 2019 13:20:16 -0500
Subject: [PATCH 007/459] Make CodeObject.source_path an Option<String>

---
 src/bytecode.rs |  4 ++--
 src/compile.rs  | 27 +++++++++++++++++----------
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/src/bytecode.rs b/src/bytecode.rs
index c87bc0ae..b5b3c11d 100644
--- a/src/bytecode.rs
+++ b/src/bytecode.rs
@@ -21,7 +21,7 @@ pub struct CodeObject {
     pub varargs: Varargs,       // *args or *
     pub kwonlyarg_names: Vec<String>,
     pub varkeywords: Varargs, // **kwargs or **
-    pub source_path: String,
+    pub source_path: Option<String>,
     pub first_line_number: usize,
     pub obj_name: String, // Name of the object that created this code object
     pub is_generator: bool,
@@ -269,7 +269,7 @@ impl CodeObject {
         varargs: Varargs,
         kwonlyarg_names: Vec<String>,
         varkeywords: Varargs,
-        source_path: String,
+        source_path: Option<String>,
         first_line_number: usize,
         obj_name: String,
     ) -> CodeObject {
diff --git a/src/compile.rs b/src/compile.rs
index 8b806b1d..0efa3adf 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -23,7 +23,11 @@ struct Compiler {
 }
 
 /// Compile a given sourcecode into a bytecode object.
-pub fn compile(source: &str, mode: &Mode, source_path: String) -> Result<CodeObject, CompileError> {
+pub fn compile(
+    source: &str,
+    mode: &Mode,
+    source_path: Option<String>,
+) -> Result<CodeObject, CompileError> {
     match mode {
         Mode::Exec => {
             let ast = parser::parse_program(source)?;
@@ -42,11 +46,11 @@ pub fn compile(source: &str, mode: &Mode, source_path: String) -> Result<CodeObj
 
 /// A helper function for the shared code of the different compile functions
 fn with_compiler(
-    source_path: String,
+    source_path: Option<String>,
     f: impl FnOnce(&mut Compiler) -> Result<(), CompileError>,
 ) -> Result<CodeObject, CompileError> {
     let mut compiler = Compiler::new();
-    compiler.source_path = Some(source_path);
+    compiler.source_path = source_path;
     compiler.push_new_code_object("<module>".to_string());
     f(&mut compiler)?;
     let code = compiler.pop_code_object();
@@ -55,7 +59,10 @@ fn with_compiler(
 }
 
 /// Compile a standard Python program to bytecode
-pub fn compile_program(ast: ast::Program, source_path: String) -> Result<CodeObject, CompileError> {
+pub fn compile_program(
+    ast: ast::Program,
+    source_path: Option<String>,
+) -> Result<CodeObject, CompileError> {
     with_compiler(source_path, |compiler| {
         let symbol_table = make_symbol_table(&ast)?;
         compiler.compile_program(&ast, symbol_table)
@@ -65,7 +72,7 @@ pub fn compile_program(ast: ast::Program, source_path: String) -> Result<CodeObj
 /// Compile a single Python expression to bytecode
 pub fn compile_statement_eval(
     statement: Vec<ast::LocatedStatement>,
-    source_path: String,
+    source_path: Option<String>,
 ) -> Result<CodeObject, CompileError> {
     with_compiler(source_path, |compiler| {
         let symbol_table = statements_to_symbol_table(&statement)?;
@@ -76,7 +83,7 @@ pub fn compile_statement_eval(
 /// Compile a Python program to bytecode for the context of a REPL
 pub fn compile_program_single(
     ast: ast::Program,
-    source_path: String,
+    source_path: Option<String>,
 ) -> Result<CodeObject, CompileError> {
     with_compiler(source_path, |compiler| {
         let symbol_table = make_symbol_table(&ast)?;
@@ -119,7 +126,7 @@ impl Compiler {
             Varargs::None,
             Vec::new(),
             Varargs::None,
-            self.source_path.clone().unwrap(),
+            self.source_path.clone(),
             line_number,
             obj_name,
         ));
@@ -596,7 +603,7 @@ impl Compiler {
             Varargs::from(&args.vararg),
             args.kwonlyargs.iter().map(|a| a.arg.clone()).collect(),
             Varargs::from(&args.kwarg),
-            self.source_path.clone().unwrap(),
+            self.source_path.clone(),
             line_number,
             name.to_string(),
         ));
@@ -849,7 +856,7 @@ impl Compiler {
             Varargs::None,
             vec![],
             Varargs::None,
-            self.source_path.clone().unwrap(),
+            self.source_path.clone(),
             line_number,
             name.to_string(),
         ));
@@ -1571,7 +1578,7 @@ impl Compiler {
             Varargs::None,
             vec![],
             Varargs::None,
-            self.source_path.clone().unwrap(),
+            self.source_path.clone(),
             line_number,
             name.clone(),
         ));

From a6808efab70e665f1203605277b7e110ce3630d1 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Thu, 20 Jun 2019 12:01:20 -0500
Subject: [PATCH 008/459] Revert "Make CodeObject.source_path an
 Option<String>"

---
 src/bytecode.rs |  4 ++--
 src/compile.rs  | 27 ++++++++++-----------------
 2 files changed, 12 insertions(+), 19 deletions(-)

diff --git a/src/bytecode.rs b/src/bytecode.rs
index b5b3c11d..c87bc0ae 100644
--- a/src/bytecode.rs
+++ b/src/bytecode.rs
@@ -21,7 +21,7 @@ pub struct CodeObject {
     pub varargs: Varargs,       // *args or *
     pub kwonlyarg_names: Vec<String>,
     pub varkeywords: Varargs, // **kwargs or **
-    pub source_path: Option<String>,
+    pub source_path: String,
     pub first_line_number: usize,
     pub obj_name: String, // Name of the object that created this code object
     pub is_generator: bool,
@@ -269,7 +269,7 @@ impl CodeObject {
         varargs: Varargs,
         kwonlyarg_names: Vec<String>,
         varkeywords: Varargs,
-        source_path: Option<String>,
+        source_path: String,
         first_line_number: usize,
         obj_name: String,
     ) -> CodeObject {
diff --git a/src/compile.rs b/src/compile.rs
index 0efa3adf..8b806b1d 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -23,11 +23,7 @@ struct Compiler {
 }
 
 /// Compile a given sourcecode into a bytecode object.
-pub fn compile(
-    source: &str,
-    mode: &Mode,
-    source_path: Option<String>,
-) -> Result<CodeObject, CompileError> {
+pub fn compile(source: &str, mode: &Mode, source_path: String) -> Result<CodeObject, CompileError> {
     match mode {
         Mode::Exec => {
             let ast = parser::parse_program(source)?;
@@ -46,11 +42,11 @@ pub fn compile(
 
 /// A helper function for the shared code of the different compile functions
 fn with_compiler(
-    source_path: Option<String>,
+    source_path: String,
     f: impl FnOnce(&mut Compiler) -> Result<(), CompileError>,
 ) -> Result<CodeObject, CompileError> {
     let mut compiler = Compiler::new();
-    compiler.source_path = source_path;
+    compiler.source_path = Some(source_path);
     compiler.push_new_code_object("<module>".to_string());
     f(&mut compiler)?;
     let code = compiler.pop_code_object();
@@ -59,10 +55,7 @@ fn with_compiler(
 }
 
 /// Compile a standard Python program to bytecode
-pub fn compile_program(
-    ast: ast::Program,
-    source_path: Option<String>,
-) -> Result<CodeObject, CompileError> {
+pub fn compile_program(ast: ast::Program, source_path: String) -> Result<CodeObject, CompileError> {
     with_compiler(source_path, |compiler| {
         let symbol_table = make_symbol_table(&ast)?;
         compiler.compile_program(&ast, symbol_table)
@@ -72,7 +65,7 @@ pub fn compile_program(
 /// Compile a single Python expression to bytecode
 pub fn compile_statement_eval(
     statement: Vec<ast::LocatedStatement>,
-    source_path: Option<String>,
+    source_path: String,
 ) -> Result<CodeObject, CompileError> {
     with_compiler(source_path, |compiler| {
         let symbol_table = statements_to_symbol_table(&statement)?;
@@ -83,7 +76,7 @@ pub fn compile_statement_eval(
 /// Compile a Python program to bytecode for the context of a REPL
 pub fn compile_program_single(
     ast: ast::Program,
-    source_path: Option<String>,
+    source_path: String,
 ) -> Result<CodeObject, CompileError> {
     with_compiler(source_path, |compiler| {
         let symbol_table = make_symbol_table(&ast)?;
@@ -126,7 +119,7 @@ impl Compiler {
             Varargs::None,
             Vec::new(),
             Varargs::None,
-            self.source_path.clone(),
+            self.source_path.clone().unwrap(),
             line_number,
             obj_name,
         ));
@@ -603,7 +596,7 @@ impl Compiler {
             Varargs::from(&args.vararg),
             args.kwonlyargs.iter().map(|a| a.arg.clone()).collect(),
             Varargs::from(&args.kwarg),
-            self.source_path.clone(),
+            self.source_path.clone().unwrap(),
             line_number,
             name.to_string(),
         ));
@@ -856,7 +849,7 @@ impl Compiler {
             Varargs::None,
             vec![],
             Varargs::None,
-            self.source_path.clone(),
+            self.source_path.clone().unwrap(),
             line_number,
             name.to_string(),
         ));
@@ -1578,7 +1571,7 @@ impl Compiler {
             Varargs::None,
             vec![],
             Varargs::None,
-            self.source_path.clone(),
+            self.source_path.clone().unwrap(),
             line_number,
             name.clone(),
         ));

From 96e49f195d4d3d105bd12d9b8ee27006cee002a7 Mon Sep 17 00:00:00 2001
From: Aviv Palivoda <palaviv@gmail.com>
Date: Sat, 29 Jun 2019 17:04:31 +0300
Subject: [PATCH 009/459] Calculate import level at parsing

---
 src/bytecode.rs | 20 ++++++++++++++++++--
 src/compile.rs  |  6 ++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/src/bytecode.rs b/src/bytecode.rs
index c87bc0ae..e41441c8 100644
--- a/src/bytecode.rs
+++ b/src/bytecode.rs
@@ -51,9 +51,11 @@ pub enum Instruction {
     Import {
         name: String,
         symbols: Vec<String>,
+        level: usize,
     },
     ImportStar {
         name: String,
+        level: usize,
     },
     LoadName {
         name: String,
@@ -327,11 +329,25 @@ impl Instruction {
             ($variant:ident, $var1:expr, $var2:expr) => {
                 write!(f, "{:20} ({}, {})\n", stringify!($variant), $var1, $var2)
             };
+            ($variant:ident, $var1:expr, $var2:expr, $var3:expr) => {
+                write!(
+                    f,
+                    "{:20} ({}, {}, {})\n",
+                    stringify!($variant),
+                    $var1,
+                    $var2,
+                    $var3
+                )
+            };
         }
 
         match self {
-            Import { name, symbols } => w!(Import, name, format!("{:?}", symbols)),
-            ImportStar { name } => w!(ImportStar, name),
+            Import {
+                name,
+                symbols,
+                level,
+            } => w!(Import, name, format!("{:?}", symbols), level),
+            ImportStar { name, level } => w!(ImportStar, name, level),
             LoadName { name, scope } => w!(LoadName, name, format!("{:?}", scope)),
             StoreName { name, scope } => w!(StoreName, name, format!("{:?}", scope)),
             DeleteName { name } => w!(DeleteName, name),
diff --git a/src/compile.rs b/src/compile.rs
index 8b806b1d..c4e4a1a2 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -252,13 +252,16 @@ impl Compiler {
                     module,
                     symbols,
                     alias,
+                    level,
                 } in import_parts
                 {
+                    let level = *level;
                     if let Some(alias) = alias {
                         // import module as alias
                         self.emit(Instruction::Import {
                             name: module.clone(),
                             symbols: vec![],
+                            level,
                         });
                         self.store_name(&alias);
                     } else if symbols.is_empty() {
@@ -266,6 +269,7 @@ impl Compiler {
                         self.emit(Instruction::Import {
                             name: module.clone(),
                             symbols: vec![],
+                            level,
                         });
                         self.store_name(&module.clone());
                     } else {
@@ -276,6 +280,7 @@ impl Compiler {
                             // from module import *
                             self.emit(Instruction::ImportStar {
                                 name: module.clone(),
+                                level,
                             });
                         } else {
                             // from module import symbol
@@ -292,6 +297,7 @@ impl Compiler {
                             self.emit(Instruction::Import {
                                 name: module.clone(),
                                 symbols: symbols_strings,
+                                level,
                             });
                             names.iter().rev().for_each(|name| self.store_name(&name));
                         }

From 134e6b2579707405e5bb9f946441215bfe627e0a Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Sun, 30 Jun 2019 11:01:40 +0200
Subject: [PATCH 010/459] Move bytecode into own crate.

---
 Cargo.toml      |   3 +-
 src/bytecode.rs | 458 ------------------------------------------------
 src/compile.rs  |   2 +-
 src/lib.rs      |   1 -
 4 files changed, 3 insertions(+), 461 deletions(-)
 delete mode 100644 src/bytecode.rs

diff --git a/Cargo.toml b/Cargo.toml
index fc0d1959..d0aa0926 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,11 +1,12 @@
 [package]
 name = "rustpython_compiler"
 version = "0.1.0"
-authors = ["coolreader18 <33094578+coolreader18@users.noreply.github.com>"]
+authors = ["coolreader18 <33094578+coolreader18@users.noreply.github.com>", "Windel Bouwman <windel@windel.nl>"]
 edition = "2018"
 
 [dependencies]
 bitflags = "1.1"
+rustpython_bytecode = { path = "../bytecode" }
 rustpython_parser = { path = "../parser" }
 serde = { version = "1.0", features = ["derive"] }
 num-complex = { version = "0.2", features = ["serde"] }
diff --git a/src/bytecode.rs b/src/bytecode.rs
deleted file mode 100644
index e41441c8..00000000
--- a/src/bytecode.rs
+++ /dev/null
@@ -1,458 +0,0 @@
-//! Implement python as a virtual machine with bytecodes. This module
-//! implements bytecode structure.
-
-use bitflags::bitflags;
-use num_bigint::BigInt;
-use num_complex::Complex64;
-use rustpython_parser::ast;
-use serde::{Deserialize, Serialize};
-use std::collections::{HashMap, HashSet};
-use std::fmt;
-
-/// Primary container of a single code object. Each python function has
-/// a codeobject. Also a module has a codeobject.
-#[derive(Clone, PartialEq, Serialize, Deserialize)]
-pub struct CodeObject {
-    pub instructions: Vec<Instruction>,
-    /// Jump targets.
-    pub label_map: HashMap<Label, usize>,
-    pub locations: Vec<ast::Location>,
-    pub arg_names: Vec<String>, // Names of positional arguments
-    pub varargs: Varargs,       // *args or *
-    pub kwonlyarg_names: Vec<String>,
-    pub varkeywords: Varargs, // **kwargs or **
-    pub source_path: String,
-    pub first_line_number: usize,
-    pub obj_name: String, // Name of the object that created this code object
-    pub is_generator: bool,
-}
-
-bitflags! {
-    #[derive(Serialize, Deserialize)]
-    pub struct FunctionOpArg: u8 {
-        const HAS_DEFAULTS = 0x01;
-        const HAS_KW_ONLY_DEFAULTS = 0x02;
-        const HAS_ANNOTATIONS = 0x04;
-    }
-}
-
-pub type Label = usize;
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum NameScope {
-    Local,
-    NonLocal,
-    Global,
-}
-
-/// A Single bytecode instruction.
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum Instruction {
-    Import {
-        name: String,
-        symbols: Vec<String>,
-        level: usize,
-    },
-    ImportStar {
-        name: String,
-        level: usize,
-    },
-    LoadName {
-        name: String,
-        scope: NameScope,
-    },
-    StoreName {
-        name: String,
-        scope: NameScope,
-    },
-    DeleteName {
-        name: String,
-    },
-    StoreSubscript,
-    DeleteSubscript,
-    StoreAttr {
-        name: String,
-    },
-    DeleteAttr {
-        name: String,
-    },
-    LoadConst {
-        value: Constant,
-    },
-    UnaryOperation {
-        op: UnaryOperator,
-    },
-    BinaryOperation {
-        op: BinaryOperator,
-        inplace: bool,
-    },
-    LoadAttr {
-        name: String,
-    },
-    CompareOperation {
-        op: ComparisonOperator,
-    },
-    Pop,
-    Rotate {
-        amount: usize,
-    },
-    Duplicate,
-    GetIter,
-    Pass,
-    Continue,
-    Break,
-    Jump {
-        target: Label,
-    },
-    JumpIf {
-        target: Label,
-    },
-    JumpIfFalse {
-        target: Label,
-    },
-    MakeFunction {
-        flags: FunctionOpArg,
-    },
-    CallFunction {
-        typ: CallType,
-    },
-    ForIter {
-        target: Label,
-    },
-    ReturnValue,
-    YieldValue,
-    YieldFrom,
-    SetupLoop {
-        start: Label,
-        end: Label,
-    },
-    SetupExcept {
-        handler: Label,
-    },
-    SetupWith {
-        end: Label,
-    },
-    CleanupWith {
-        end: Label,
-    },
-    PopBlock,
-    Raise {
-        argc: usize,
-    },
-    BuildString {
-        size: usize,
-    },
-    BuildTuple {
-        size: usize,
-        unpack: bool,
-    },
-    BuildList {
-        size: usize,
-        unpack: bool,
-    },
-    BuildSet {
-        size: usize,
-        unpack: bool,
-    },
-    BuildMap {
-        size: usize,
-        unpack: bool,
-    },
-    BuildSlice {
-        size: usize,
-    },
-    ListAppend {
-        i: usize,
-    },
-    SetAdd {
-        i: usize,
-    },
-    MapAdd {
-        i: usize,
-    },
-    PrintExpr,
-    LoadBuildClass,
-    UnpackSequence {
-        size: usize,
-    },
-    UnpackEx {
-        before: usize,
-        after: usize,
-    },
-    Unpack,
-    FormatValue {
-        conversion: Option<ast::ConversionFlag>,
-        spec: String,
-    },
-    PopException,
-}
-
-use self::Instruction::*;
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum CallType {
-    Positional(usize),
-    Keyword(usize),
-    Ex(bool),
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum Constant {
-    Integer { value: BigInt },
-    Float { value: f64 },
-    Complex { value: Complex64 },
-    Boolean { value: bool },
-    String { value: String },
-    Bytes { value: Vec<u8> },
-    Code { code: Box<CodeObject> },
-    Tuple { elements: Vec<Constant> },
-    None,
-    Ellipsis,
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum ComparisonOperator {
-    Greater,
-    GreaterOrEqual,
-    Less,
-    LessOrEqual,
-    Equal,
-    NotEqual,
-    In,
-    NotIn,
-    Is,
-    IsNot,
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum BinaryOperator {
-    Power,
-    Multiply,
-    MatrixMultiply,
-    Divide,
-    FloorDivide,
-    Modulo,
-    Add,
-    Subtract,
-    Subscript,
-    Lshift,
-    Rshift,
-    And,
-    Xor,
-    Or,
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum UnaryOperator {
-    Not,
-    Invert,
-    Minus,
-    Plus,
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub enum Varargs {
-    None,
-    Unnamed,
-    Named(String),
-}
-
-/*
-Maintain a stack of blocks on the VM.
-pub enum BlockType {
-    Loop,
-    Except,
-}
-*/
-
-impl CodeObject {
-    pub fn new(
-        arg_names: Vec<String>,
-        varargs: Varargs,
-        kwonlyarg_names: Vec<String>,
-        varkeywords: Varargs,
-        source_path: String,
-        first_line_number: usize,
-        obj_name: String,
-    ) -> CodeObject {
-        CodeObject {
-            instructions: Vec::new(),
-            label_map: HashMap::new(),
-            locations: Vec::new(),
-            arg_names,
-            varargs,
-            kwonlyarg_names,
-            varkeywords,
-            source_path,
-            first_line_number,
-            obj_name,
-            is_generator: false,
-        }
-    }
-
-    pub fn get_constants(&self) -> impl Iterator<Item = &Constant> {
-        self.instructions.iter().filter_map(|x| {
-            if let Instruction::LoadConst { value } = x {
-                Some(value)
-            } else {
-                None
-            }
-        })
-    }
-}
-
-impl fmt::Display for CodeObject {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let label_targets: HashSet<&usize> = self.label_map.values().collect();
-        for (offset, instruction) in self.instructions.iter().enumerate() {
-            let arrow = if label_targets.contains(&offset) {
-                ">>"
-            } else {
-                "  "
-            };
-            write!(f, "          {} {:5} ", arrow, offset)?;
-            instruction.fmt_dis(f, &self.label_map)?;
-        }
-        Ok(())
-    }
-}
-
-impl Instruction {
-    fn fmt_dis(&self, f: &mut fmt::Formatter, label_map: &HashMap<Label, usize>) -> fmt::Result {
-        macro_rules! w {
-            ($variant:ident) => {
-                write!(f, "{:20}\n", stringify!($variant))
-            };
-            ($variant:ident, $var:expr) => {
-                write!(f, "{:20} ({})\n", stringify!($variant), $var)
-            };
-            ($variant:ident, $var1:expr, $var2:expr) => {
-                write!(f, "{:20} ({}, {})\n", stringify!($variant), $var1, $var2)
-            };
-            ($variant:ident, $var1:expr, $var2:expr, $var3:expr) => {
-                write!(
-                    f,
-                    "{:20} ({}, {}, {})\n",
-                    stringify!($variant),
-                    $var1,
-                    $var2,
-                    $var3
-                )
-            };
-        }
-
-        match self {
-            Import {
-                name,
-                symbols,
-                level,
-            } => w!(Import, name, format!("{:?}", symbols), level),
-            ImportStar { name, level } => w!(ImportStar, name, level),
-            LoadName { name, scope } => w!(LoadName, name, format!("{:?}", scope)),
-            StoreName { name, scope } => w!(StoreName, name, format!("{:?}", scope)),
-            DeleteName { name } => w!(DeleteName, name),
-            StoreSubscript => w!(StoreSubscript),
-            DeleteSubscript => w!(DeleteSubscript),
-            StoreAttr { name } => w!(StoreAttr, name),
-            DeleteAttr { name } => w!(DeleteAttr, name),
-            LoadConst { value } => w!(LoadConst, value),
-            UnaryOperation { op } => w!(UnaryOperation, format!("{:?}", op)),
-            BinaryOperation { op, inplace } => w!(BinaryOperation, format!("{:?}", op), inplace),
-            LoadAttr { name } => w!(LoadAttr, name),
-            CompareOperation { op } => w!(CompareOperation, format!("{:?}", op)),
-            Pop => w!(Pop),
-            Rotate { amount } => w!(Rotate, amount),
-            Duplicate => w!(Duplicate),
-            GetIter => w!(GetIter),
-            Pass => w!(Pass),
-            Continue => w!(Continue),
-            Break => w!(Break),
-            Jump { target } => w!(Jump, label_map[target]),
-            JumpIf { target } => w!(JumpIf, label_map[target]),
-            JumpIfFalse { target } => w!(JumpIfFalse, label_map[target]),
-            MakeFunction { flags } => w!(MakeFunction, format!("{:?}", flags)),
-            CallFunction { typ } => w!(CallFunction, format!("{:?}", typ)),
-            ForIter { target } => w!(ForIter, label_map[target]),
-            ReturnValue => w!(ReturnValue),
-            YieldValue => w!(YieldValue),
-            YieldFrom => w!(YieldFrom),
-            SetupLoop { start, end } => w!(SetupLoop, label_map[start], label_map[end]),
-            SetupExcept { handler } => w!(SetupExcept, handler),
-            SetupWith { end } => w!(SetupWith, end),
-            CleanupWith { end } => w!(CleanupWith, end),
-            PopBlock => w!(PopBlock),
-            Raise { argc } => w!(Raise, argc),
-            BuildString { size } => w!(BuildString, size),
-            BuildTuple { size, unpack } => w!(BuildTuple, size, unpack),
-            BuildList { size, unpack } => w!(BuildList, size, unpack),
-            BuildSet { size, unpack } => w!(BuildSet, size, unpack),
-            BuildMap { size, unpack } => w!(BuildMap, size, unpack),
-            BuildSlice { size } => w!(BuildSlice, size),
-            ListAppend { i } => w!(ListAppend, i),
-            SetAdd { i } => w!(SetAdd, i),
-            MapAdd { i } => w!(MapAdd, i),
-            PrintExpr => w!(PrintExpr),
-            LoadBuildClass => w!(LoadBuildClass),
-            UnpackSequence { size } => w!(UnpackSequence, size),
-            UnpackEx { before, after } => w!(UnpackEx, before, after),
-            Unpack => w!(Unpack),
-            FormatValue { spec, .. } => w!(FormatValue, spec), // TODO: write conversion
-            PopException => w!(PopException),
-        }
-    }
-}
-
-impl fmt::Display for Constant {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            Constant::Integer { value } => write!(f, "{}", value),
-            Constant::Float { value } => write!(f, "{}", value),
-            Constant::Complex { value } => write!(f, "{}", value),
-            Constant::Boolean { value } => write!(f, "{}", value),
-            Constant::String { value } => write!(f, "{:?}", value),
-            Constant::Bytes { value } => write!(f, "{:?}", value),
-            Constant::Code { code } => write!(f, "{:?}", code),
-            Constant::Tuple { elements } => write!(
-                f,
-                "({})",
-                elements
-                    .iter()
-                    .map(|e| format!("{}", e))
-                    .collect::<Vec<_>>()
-                    .join(", ")
-            ),
-            Constant::None => write!(f, "None"),
-            Constant::Ellipsis => write!(f, "Ellipsis"),
-        }
-    }
-}
-
-impl fmt::Debug for CodeObject {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "<code object {} at ??? file {:?}, line {}>",
-            self.obj_name, self.source_path, self.first_line_number
-        )
-    }
-}
-
-impl From<ast::Varargs> for Varargs {
-    fn from(varargs: ast::Varargs) -> Varargs {
-        match varargs {
-            ast::Varargs::None => Varargs::None,
-            ast::Varargs::Unnamed => Varargs::Unnamed,
-            ast::Varargs::Named(param) => Varargs::Named(param.arg),
-        }
-    }
-}
-
-impl<'a> From<&'a ast::Varargs> for Varargs {
-    fn from(varargs: &'a ast::Varargs) -> Varargs {
-        match varargs {
-            ast::Varargs::None => Varargs::None,
-            ast::Varargs::Unnamed => Varargs::Unnamed,
-            ast::Varargs::Named(ref param) => Varargs::Named(param.arg.clone()),
-        }
-    }
-}
diff --git a/src/compile.rs b/src/compile.rs
index c4e4a1a2..b50ae332 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -5,10 +5,10 @@
 //!   https://github.com/python/cpython/blob/master/Python/compile.c
 //!   https://github.com/micropython/micropython/blob/master/py/compile.c
 
-use crate::bytecode::{self, CallType, CodeObject, Instruction, Varargs};
 use crate::error::{CompileError, CompileErrorType};
 use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolRole, SymbolScope};
 use num_complex::Complex64;
+use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Varargs};
 use rustpython_parser::{ast, parser};
 
 struct Compiler {
diff --git a/src/lib.rs b/src/lib.rs
index 382647d1..e0e6fb2b 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -4,7 +4,6 @@
 #[macro_use]
 extern crate log;
 
-pub mod bytecode;
 pub mod compile;
 pub mod error;
 mod symboltable;

From 57b4d432efb3e924bc3d168122aeb7eae36c6a2d Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Sun, 30 Jun 2019 11:42:36 +0200
Subject: [PATCH 011/459] Make bytecode crate independent of parser crate.

---
 Cargo.toml     |  3 ---
 src/compile.rs | 34 +++++++++++++++++++++++++++-------
 2 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index d0aa0926..dccc3b07 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -5,10 +5,7 @@ authors = ["coolreader18 <33094578+coolreader18@users.noreply.github.com>", "Win
 edition = "2018"
 
 [dependencies]
-bitflags = "1.1"
 rustpython_bytecode = { path = "../bytecode" }
 rustpython_parser = { path = "../parser" }
-serde = { version = "1.0", features = ["derive"] }
 num-complex = { version = "0.2", features = ["serde"] }
-num-bigint = { version = "0.2", features = ["serde"] }
 log = "0.3"
diff --git a/src/compile.rs b/src/compile.rs
index b50ae332..7ade38bc 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -599,9 +599,9 @@ impl Compiler {
         let line_number = self.get_source_line_number();
         self.code_object_stack.push(CodeObject::new(
             args.args.iter().map(|a| a.arg.clone()).collect(),
-            Varargs::from(&args.vararg),
+            compile_varargs(&args.vararg),
             args.kwonlyargs.iter().map(|a| a.arg.clone()).collect(),
-            Varargs::from(&args.kwarg),
+            compile_varargs(&args.kwarg),
             self.source_path.clone().unwrap(),
             line_number,
             name.to_string(),
@@ -1743,7 +1743,7 @@ impl Compiler {
             } => {
                 self.compile_expression(value)?;
                 self.emit(Instruction::FormatValue {
-                    conversion: *conversion,
+                    conversion: conversion.map(compile_conversion_flag),
                     spec: spec.clone(),
                 });
             }
@@ -1773,7 +1773,7 @@ impl Compiler {
 
     // Low level helper functions:
     fn emit(&mut self, instruction: Instruction) {
-        let location = self.current_source_location.clone();
+        let location = compile_location(&self.current_source_location);
         let cur_code_obj = self.current_code_object();
         cur_code_obj.instructions.push(instruction);
         cur_code_obj.locations.push(location);
@@ -1834,13 +1834,33 @@ fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option<
     (body, None)
 }
 
+fn compile_location(location: &ast::Location) -> bytecode::Location {
+    bytecode::Location::new(location.get_row(), location.get_column())
+}
+
+fn compile_varargs(varargs: &ast::Varargs) -> bytecode::Varargs {
+    match varargs {
+        ast::Varargs::None => bytecode::Varargs::None,
+        ast::Varargs::Unnamed => bytecode::Varargs::Unnamed,
+        ast::Varargs::Named(param) => bytecode::Varargs::Named(param.arg.clone()),
+    }
+}
+
+fn compile_conversion_flag(conversion_flag: ast::ConversionFlag) -> bytecode::ConversionFlag {
+    match conversion_flag {
+        ast::ConversionFlag::Ascii => bytecode::ConversionFlag::Ascii,
+        ast::ConversionFlag::Repr => bytecode::ConversionFlag::Repr,
+        ast::ConversionFlag::Str => bytecode::ConversionFlag::Str,
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::Compiler;
-    use crate::bytecode::CodeObject;
-    use crate::bytecode::Constant::*;
-    use crate::bytecode::Instruction::*;
     use crate::symboltable::make_symbol_table;
+    use rustpython_bytecode::bytecode::CodeObject;
+    use rustpython_bytecode::bytecode::Constant::*;
+    use rustpython_bytecode::bytecode::Instruction::*;
     use rustpython_parser::parser;
 
     fn compile_exec(source: &str) -> CodeObject {

From 6f063a402d3f5d2270f6462642f719f6103a0675 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Sun, 30 Jun 2019 20:23:01 +0200
Subject: [PATCH 012/459] Change get_row() into row()

---
 src/compile.rs | 4 ++--
 src/error.rs   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 7ade38bc..844f952e 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1803,7 +1803,7 @@ impl Compiler {
     }
 
     fn get_source_line_number(&mut self) -> usize {
-        self.current_source_location.get_row()
+        self.current_source_location.row()
     }
 
     fn create_qualified_name(&self, name: &str, suffix: &str) -> String {
@@ -1835,7 +1835,7 @@ fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option<
 }
 
 fn compile_location(location: &ast::Location) -> bytecode::Location {
-    bytecode::Location::new(location.get_row(), location.get_column())
+    bytecode::Location::new(location.row(), location.column())
 }
 
 fn compile_varargs(varargs: &ast::Varargs) -> bytecode::Varargs {
diff --git a/src/error.rs b/src/error.rs
index af763168..bd9950c8 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -56,7 +56,7 @@ impl fmt::Display for CompileError {
         }?;
 
         // Print line number:
-        write!(f, " at line {:?}", self.location.get_row())
+        write!(f, " at line {:?}", self.location.row())
     }
 }
 

From a8de4044b86b42a869d0066913564ba695a514c1 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Mon, 1 Jul 2019 21:05:29 +0200
Subject: [PATCH 013/459] Change underscore into hyphen

---
 Cargo.toml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index dccc3b07..9056260a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,11 +1,11 @@
 [package]
-name = "rustpython_compiler"
+name = "rustpython-compiler"
 version = "0.1.0"
 authors = ["coolreader18 <33094578+coolreader18@users.noreply.github.com>", "Windel Bouwman <windel@windel.nl>"]
 edition = "2018"
 
 [dependencies]
-rustpython_bytecode = { path = "../bytecode" }
-rustpython_parser = { path = "../parser" }
+rustpython-bytecode = { path = "../bytecode" }
+rustpython-parser = { path = "../parser" }
 num-complex = { version = "0.2", features = ["serde"] }
 log = "0.3"

From 4396af76fbc5948e9a322f9423113875f424cb75 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Mon, 1 Jul 2019 21:14:07 +0200
Subject: [PATCH 014/459] Change authors to team name.

---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 9056260a..8fdb3787 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "rustpython-compiler"
 version = "0.1.0"
-authors = ["coolreader18 <33094578+coolreader18@users.noreply.github.com>", "Windel Bouwman <windel@windel.nl>"]
+authors = ["RustPython Team"]
 edition = "2018"
 
 [dependencies]

From 313ffe6d6a9b38e745ed6ff0295eced115027f23 Mon Sep 17 00:00:00 2001
From: Maxim Kurnikov <maxim.kurnikov@gmail.com>
Date: Wed, 3 Jul 2019 16:13:16 +0300
Subject: [PATCH 015/459] Do not pollute stack when if-expression condition
 evaluated to False

---
 src/compile.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index 844f952e..877200f8 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1426,11 +1426,15 @@ impl Compiler {
             ast::Expression::IfExpression { test, body, orelse } => {
                 let no_label = self.new_label();
                 let end_label = self.new_label();
-                self.compile_test(test, None, Some(no_label), EvalContext::Expression)?;
+                self.compile_test(test, None, None, EvalContext::Expression)?;
+                self.emit(Instruction::JumpIfFalse { target: no_label });
+                // True case
                 self.compile_expression(body)?;
                 self.emit(Instruction::Jump { target: end_label });
+                // False case
                 self.set_label(no_label);
                 self.compile_expression(orelse)?;
+                // End
                 self.set_label(end_label);
             }
         }

From f4a94903b3dab1088f8317c390c834e48fcb2aa5 Mon Sep 17 00:00:00 2001
From: Aviv Palivoda <palaviv@gmail.com>
Date: Fri, 5 Jul 2019 11:01:49 +0300
Subject: [PATCH 016/459] Pass more information in user defined parse error

---
 src/error.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/error.rs b/src/error.rs
index bd9950c8..a46b915c 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -56,7 +56,10 @@ impl fmt::Display for CompileError {
         }?;
 
         // Print line number:
-        write!(f, " at line {:?}", self.location.row())
+        match &self.error {
+            CompileErrorType::Parse(..) => Ok(()),
+            _ => write!(f, " at line {:?}", self.location.row()),
+        }
     }
 }
 

From 9f068a31466ef7e2650c91cab1edaee037d9fd2b Mon Sep 17 00:00:00 2001
From: Aviv Palivoda <palaviv@gmail.com>
Date: Sat, 6 Jul 2019 10:23:32 +0300
Subject: [PATCH 017/459] Pass ParseError location to CompileError

---
 src/error.rs | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/error.rs b/src/error.rs
index a46b915c..b0e631b5 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -1,4 +1,4 @@
-use rustpython_parser::error::ParseError;
+use rustpython_parser::error::{ParseError, ParseErrorType};
 use rustpython_parser::lexer::Location;
 
 use std::error::Error;
@@ -13,8 +13,8 @@ pub struct CompileError {
 impl From<ParseError> for CompileError {
     fn from(error: ParseError) -> Self {
         CompileError {
-            error: CompileErrorType::Parse(error),
-            location: Default::default(), // TODO: extract location from parse error!
+            error: CompileErrorType::Parse(error.error),
+            location: error.location,
         }
     }
 }
@@ -28,7 +28,7 @@ pub enum CompileErrorType {
     /// Expected an expression got a statement
     ExpectExpr,
     /// Parser error
-    Parse(ParseError),
+    Parse(ParseErrorType),
     SyntaxError(String),
     /// Multiple `*` detected
     StarArgs,
@@ -56,10 +56,7 @@ impl fmt::Display for CompileError {
         }?;
 
         // Print line number:
-        match &self.error {
-            CompileErrorType::Parse(..) => Ok(()),
-            _ => write!(f, " at line {:?}", self.location.row()),
-        }
+        write!(f, " at {}", self.location)
     }
 }
 

From 888cb310e5facc479b161951f838badd889d788f Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Sat, 6 Jul 2019 20:18:18 +0200
Subject: [PATCH 018/459] parser error handling improvements.

---
 src/error.rs       | 2 +-
 src/symboltable.rs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/error.rs b/src/error.rs
index b0e631b5..679a914b 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -1,5 +1,5 @@
 use rustpython_parser::error::{ParseError, ParseErrorType};
-use rustpython_parser::lexer::Location;
+use rustpython_parser::location::Location;
 
 use std::error::Error;
 use std::fmt;
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 04e5d9ed..961f0282 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -9,7 +9,7 @@ Inspirational file: https://github.com/python/cpython/blob/master/Python/symtabl
 
 use crate::error::{CompileError, CompileErrorType};
 use rustpython_parser::ast;
-use rustpython_parser::lexer::Location;
+use rustpython_parser::location::Location;
 use std::collections::HashMap;
 
 pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolScope, SymbolTableError> {

From 71cd8cad85e3dc049118e21c7194c5597bd2fa27 Mon Sep 17 00:00:00 2001
From: Maxim Kurnikov <maxim.kurnikov@gmail.com>
Date: Sun, 7 Jul 2019 06:27:07 +0300
Subject: [PATCH 019/459] fix remaining clippy warnings

---
 src/symboltable.rs | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 04e5d9ed..2e16fdae 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -301,19 +301,17 @@ impl SymbolTableBuilder {
                     if let Some(alias) = &part.alias {
                         // `import mymodule as myalias`
                         self.register_name(alias, SymbolRole::Assigned)?;
+                    } else if part.symbols.is_empty() {
+                        // `import module`
+                        self.register_name(&part.module, SymbolRole::Assigned)?;
                     } else {
-                        if part.symbols.is_empty() {
-                            // `import module`
-                            self.register_name(&part.module, SymbolRole::Assigned)?;
-                        } else {
-                            // `from mymodule import myimport`
-                            for symbol in &part.symbols {
-                                if let Some(alias) = &symbol.alias {
-                                    // `from mymodule import myimportname as myalias`
-                                    self.register_name(alias, SymbolRole::Assigned)?;
-                                } else {
-                                    self.register_name(&symbol.symbol, SymbolRole::Assigned)?;
-                                }
+                        // `from mymodule import myimport`
+                        for symbol in &part.symbols {
+                            if let Some(alias) = &symbol.alias {
+                                // `from mymodule import myimportname as myalias`
+                                self.register_name(alias, SymbolRole::Assigned)?;
+                            } else {
+                                self.register_name(&symbol.symbol, SymbolRole::Assigned)?;
                             }
                         }
                     }

From 6e759ac5bce86e983c0d12f7c93626dad7e91c00 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Sun, 7 Jul 2019 13:04:12 +0200
Subject: [PATCH 020/459] Set all versions to 0.1.0

---
 Cargo.toml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 8fdb3787..0a2c155c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,11 +1,14 @@
 [package]
 name = "rustpython-compiler"
 version = "0.1.0"
+description = "Compiler for python code into bytecode for the rustpython VM."
 authors = ["RustPython Team"]
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
 edition = "2018"
 
 [dependencies]
-rustpython-bytecode = { path = "../bytecode" }
-rustpython-parser = { path = "../parser" }
+rustpython-bytecode = { path = "../bytecode", version = "0.1.0" }
+rustpython-parser = { path = "../parser", version = "0.1.0" }
 num-complex = { version = "0.2", features = ["serde"] }
 log = "0.3"

From 6c0338616252b89883d818a232021ae8da02f42b Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 8 Jul 2019 23:57:18 -0500
Subject: [PATCH 021/459] Add #![doc(html_logo_url = ...)] to crates

---
 src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib.rs b/src/lib.rs
index e0e6fb2b..5e844b2e 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,5 +1,7 @@
 //! Compile a Python AST or source code into bytecode consumable by RustPython or
 //! (eventually) CPython.
+#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/master/logo.png")]
+#![doc(html_root_url = "https://docs.rs/rustpython-compiler/")]
 
 #[macro_use]
 extern crate log;

From de5e71a9db7d3628e13ee1dc396450b4ab712ba2 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Fri, 12 Jul 2019 18:41:25 +0200
Subject: [PATCH 022/459] Add optimize flag to ignore assert statements.

---
 src/compile.rs | 85 +++++++++++++++++++++++++++++++-------------------
 1 file changed, 53 insertions(+), 32 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 877200f8..f78a25b4 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -20,22 +20,28 @@ struct Compiler {
     current_qualified_path: Option<String>,
     in_loop: bool,
     in_function_def: bool,
+    optimize: bool,
 }
 
 /// Compile a given sourcecode into a bytecode object.
-pub fn compile(source: &str, mode: &Mode, source_path: String) -> Result<CodeObject, CompileError> {
+pub fn compile(
+    source: &str,
+    mode: &Mode,
+    source_path: String,
+    optimize: bool,
+) -> Result<CodeObject, CompileError> {
     match mode {
         Mode::Exec => {
             let ast = parser::parse_program(source)?;
-            compile_program(ast, source_path)
+            compile_program(ast, source_path, optimize)
         }
         Mode::Eval => {
             let statement = parser::parse_statement(source)?;
-            compile_statement_eval(statement, source_path)
+            compile_statement_eval(statement, source_path, optimize)
         }
         Mode::Single => {
             let ast = parser::parse_program(source)?;
-            compile_program_single(ast, source_path)
+            compile_program_single(ast, source_path, optimize)
         }
     }
 }
@@ -43,9 +49,10 @@ pub fn compile(source: &str, mode: &Mode, source_path: String) -> Result<CodeObj
 /// A helper function for the shared code of the different compile functions
 fn with_compiler(
     source_path: String,
+    optimize: bool,
     f: impl FnOnce(&mut Compiler) -> Result<(), CompileError>,
 ) -> Result<CodeObject, CompileError> {
-    let mut compiler = Compiler::new();
+    let mut compiler = Compiler::new(optimize);
     compiler.source_path = Some(source_path);
     compiler.push_new_code_object("<module>".to_string());
     f(&mut compiler)?;
@@ -55,8 +62,12 @@ fn with_compiler(
 }
 
 /// Compile a standard Python program to bytecode
-pub fn compile_program(ast: ast::Program, source_path: String) -> Result<CodeObject, CompileError> {
-    with_compiler(source_path, |compiler| {
+pub fn compile_program(
+    ast: ast::Program,
+    source_path: String,
+    optimize: bool,
+) -> Result<CodeObject, CompileError> {
+    with_compiler(source_path, optimize, |compiler| {
         let symbol_table = make_symbol_table(&ast)?;
         compiler.compile_program(&ast, symbol_table)
     })
@@ -66,8 +77,9 @@ pub fn compile_program(ast: ast::Program, source_path: String) -> Result<CodeObj
 pub fn compile_statement_eval(
     statement: Vec<ast::LocatedStatement>,
     source_path: String,
+    optimize: bool,
 ) -> Result<CodeObject, CompileError> {
-    with_compiler(source_path, |compiler| {
+    with_compiler(source_path, optimize, |compiler| {
         let symbol_table = statements_to_symbol_table(&statement)?;
         compiler.compile_statement_eval(&statement, symbol_table)
     })
@@ -77,8 +89,9 @@ pub fn compile_statement_eval(
 pub fn compile_program_single(
     ast: ast::Program,
     source_path: String,
+    optimize: bool,
 ) -> Result<CodeObject, CompileError> {
-    with_compiler(source_path, |compiler| {
+    with_compiler(source_path, optimize, |compiler| {
         let symbol_table = make_symbol_table(&ast)?;
         compiler.compile_program_single(&ast, symbol_table)
     })
@@ -98,8 +111,14 @@ enum EvalContext {
 
 type Label = usize;
 
+impl Default for Compiler {
+    fn default() -> Self {
+        Compiler::new(false)
+    }
+}
+
 impl Compiler {
-    fn new() -> Self {
+    fn new(optimize: bool) -> Self {
         Compiler {
             code_object_stack: Vec::new(),
             scope_stack: Vec::new(),
@@ -109,6 +128,7 @@ impl Compiler {
             current_qualified_path: None,
             in_loop: false,
             in_function_def: false,
+            optimize,
         }
     }
 
@@ -434,29 +454,30 @@ impl Compiler {
                 decorator_list,
             } => self.compile_class_def(name, body, bases, keywords, decorator_list)?,
             ast::Statement::Assert { test, msg } => {
-                // TODO: if some flag, ignore all assert statements!
-
-                let end_label = self.new_label();
-                self.compile_test(test, Some(end_label), None, EvalContext::Statement)?;
-                self.emit(Instruction::LoadName {
-                    name: String::from("AssertionError"),
-                    scope: bytecode::NameScope::Local,
-                });
-                match msg {
-                    Some(e) => {
-                        self.compile_expression(e)?;
-                        self.emit(Instruction::CallFunction {
-                            typ: CallType::Positional(1),
-                        });
-                    }
-                    None => {
-                        self.emit(Instruction::CallFunction {
-                            typ: CallType::Positional(0),
-                        });
+                // if some flag, ignore all assert statements!
+                if !self.optimize {
+                    let end_label = self.new_label();
+                    self.compile_test(test, Some(end_label), None, EvalContext::Statement)?;
+                    self.emit(Instruction::LoadName {
+                        name: String::from("AssertionError"),
+                        scope: bytecode::NameScope::Local,
+                    });
+                    match msg {
+                        Some(e) => {
+                            self.compile_expression(e)?;
+                            self.emit(Instruction::CallFunction {
+                                typ: CallType::Positional(1),
+                            });
+                        }
+                        None => {
+                            self.emit(Instruction::CallFunction {
+                                typ: CallType::Positional(0),
+                            });
+                        }
                     }
+                    self.emit(Instruction::Raise { argc: 1 });
+                    self.set_label(end_label);
                 }
-                self.emit(Instruction::Raise { argc: 1 });
-                self.set_label(end_label);
             }
             ast::Statement::Break => {
                 if !self.in_loop {
@@ -1868,7 +1889,7 @@ mod tests {
     use rustpython_parser::parser;
 
     fn compile_exec(source: &str) -> CodeObject {
-        let mut compiler = Compiler::new();
+        let mut compiler: Compiler = Default::default();
         compiler.source_path = Some("source_path".to_string());
         compiler.push_new_code_object("<module>".to_string());
         let ast = parser::parse_program(&source.to_string()).unwrap();

From 5e468baa166078e4479fa7faa39ff73ab613639c Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Sat, 13 Jul 2019 12:23:53 +0200
Subject: [PATCH 023/459] Add a settings struct for storage of different
 settings which influence the VM.

---
 src/compile.rs | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index f78a25b4..93ea6446 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -20,7 +20,7 @@ struct Compiler {
     current_qualified_path: Option<String>,
     in_loop: bool,
     in_function_def: bool,
-    optimize: bool,
+    optimize: u8,
 }
 
 /// Compile a given sourcecode into a bytecode object.
@@ -28,7 +28,7 @@ pub fn compile(
     source: &str,
     mode: &Mode,
     source_path: String,
-    optimize: bool,
+    optimize: u8,
 ) -> Result<CodeObject, CompileError> {
     match mode {
         Mode::Exec => {
@@ -49,7 +49,7 @@ pub fn compile(
 /// A helper function for the shared code of the different compile functions
 fn with_compiler(
     source_path: String,
-    optimize: bool,
+    optimize: u8,
     f: impl FnOnce(&mut Compiler) -> Result<(), CompileError>,
 ) -> Result<CodeObject, CompileError> {
     let mut compiler = Compiler::new(optimize);
@@ -65,7 +65,7 @@ fn with_compiler(
 pub fn compile_program(
     ast: ast::Program,
     source_path: String,
-    optimize: bool,
+    optimize: u8,
 ) -> Result<CodeObject, CompileError> {
     with_compiler(source_path, optimize, |compiler| {
         let symbol_table = make_symbol_table(&ast)?;
@@ -77,7 +77,7 @@ pub fn compile_program(
 pub fn compile_statement_eval(
     statement: Vec<ast::LocatedStatement>,
     source_path: String,
-    optimize: bool,
+    optimize: u8,
 ) -> Result<CodeObject, CompileError> {
     with_compiler(source_path, optimize, |compiler| {
         let symbol_table = statements_to_symbol_table(&statement)?;
@@ -89,7 +89,7 @@ pub fn compile_statement_eval(
 pub fn compile_program_single(
     ast: ast::Program,
     source_path: String,
-    optimize: bool,
+    optimize: u8,
 ) -> Result<CodeObject, CompileError> {
     with_compiler(source_path, optimize, |compiler| {
         let symbol_table = make_symbol_table(&ast)?;
@@ -113,12 +113,12 @@ type Label = usize;
 
 impl Default for Compiler {
     fn default() -> Self {
-        Compiler::new(false)
+        Compiler::new(0)
     }
 }
 
 impl Compiler {
-    fn new(optimize: bool) -> Self {
+    fn new(optimize: u8) -> Self {
         Compiler {
             code_object_stack: Vec::new(),
             scope_stack: Vec::new(),
@@ -455,7 +455,7 @@ impl Compiler {
             } => self.compile_class_def(name, body, bases, keywords, decorator_list)?,
             ast::Statement::Assert { test, msg } => {
                 // if some flag, ignore all assert statements!
-                if !self.optimize {
+                if self.optimize == 0 {
                     let end_label = self.new_label();
                     self.compile_test(test, Some(end_label), None, EvalContext::Statement)?;
                     self.emit(Instruction::LoadName {

From 69c4d0b240eeaaefc2f5c091c6ef33904e3e12c7 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Mon, 15 Jul 2019 21:00:28 +0200
Subject: [PATCH 024/459] Simplify import AST in line with CPython.

---
 src/compile.rs     | 104 +++++++++++++++++++++++----------------------
 src/symboltable.rs |  20 +++------
 2 files changed, 58 insertions(+), 66 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 93ea6446..8cd2ab35 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -267,61 +267,63 @@ impl Compiler {
         self.set_source_location(&statement.location);
 
         match &statement.node {
-            ast::Statement::Import { import_parts } => {
-                for ast::SingleImport {
-                    module,
-                    symbols,
-                    alias,
-                    level,
-                } in import_parts
-                {
-                    let level = *level;
-                    if let Some(alias) = alias {
-                        // import module as alias
-                        self.emit(Instruction::Import {
-                            name: module.clone(),
-                            symbols: vec![],
-                            level,
-                        });
-                        self.store_name(&alias);
-                    } else if symbols.is_empty() {
-                        // import module
-                        self.emit(Instruction::Import {
-                            name: module.clone(),
-                            symbols: vec![],
-                            level,
-                        });
-                        self.store_name(&module.clone());
+            ast::Statement::Import { names } => {
+                // import a, b, c as d
+                for name in names {
+                    self.emit(Instruction::Import {
+                        name: name.symbol.clone(),
+                        symbols: vec![],
+                        level: 0,
+                    });
+
+                    if let Some(alias) = &name.alias {
+                        self.store_name(alias);
                     } else {
-                        let import_star = symbols
-                            .iter()
-                            .any(|import_symbol| import_symbol.symbol == "*");
-                        if import_star {
-                            // from module import *
-                            self.emit(Instruction::ImportStar {
-                                name: module.clone(),
-                                level,
-                            });
+                        self.store_name(&name.symbol);
+                    }
+                }
+            }
+            ast::Statement::ImportFrom {
+                level,
+                module,
+                names,
+            } => {
+                let import_star = names.iter().any(|n| n.symbol == "*");
+
+                if import_star {
+                    // from .... import *
+                    self.emit(Instruction::ImportStar {
+                        name: module.clone().unwrap(),
+                        level: *level,
+                    });
+                } else {
+                    // from mod import a, b as c
+                    // First, determine the fromlist (for import lib):
+                    let from_list = names.iter().map(|n| n.symbol.clone()).collect();
+
+                    // Load module once:
+                    self.emit(Instruction::Import {
+                        name: module.clone().unwrap(),
+                        symbols: from_list,
+                        level: *level,
+                    });
+
+                    for name in names {
+                        // import symbol from module:
+                        self.emit(Instruction::ImportFrom {
+                            name: name.symbol.to_string(),
+                        });
+
+                        // Store module under proper name:
+                        if let Some(alias) = &name.alias {
+                            self.store_name(alias);
                         } else {
-                            // from module import symbol
-                            // from module import symbol as alias
-                            let (names, symbols_strings): (Vec<String>, Vec<String>) = symbols
-                                .iter()
-                                .map(|ast::ImportSymbol { symbol, alias }| {
-                                    (
-                                        alias.clone().unwrap_or_else(|| symbol.to_string()),
-                                        symbol.to_string(),
-                                    )
-                                })
-                                .unzip();
-                            self.emit(Instruction::Import {
-                                name: module.clone(),
-                                symbols: symbols_strings,
-                                level,
-                            });
-                            names.iter().rev().for_each(|name| self.store_name(&name));
+                            self.store_name(&name.symbol);
                         }
                     }
+
+                    // Pop module from stack:
+                    self.emit(Instruction::Pop);
                 }
             }
             ast::Statement::Expression { expression } => {
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 68c5bc47..5490bd05 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -296,24 +296,14 @@ impl SymbolTableBuilder {
             ast::Statement::Break | ast::Statement::Continue | ast::Statement::Pass => {
                 // No symbols here.
             }
-            ast::Statement::Import { import_parts } => {
-                for part in import_parts {
-                    if let Some(alias) = &part.alias {
+            ast::Statement::Import { names } | ast::Statement::ImportFrom { names, .. } => {
+                for name in names {
+                    if let Some(alias) = &name.alias {
                         // `import mymodule as myalias`
                         self.register_name(alias, SymbolRole::Assigned)?;
-                    } else if part.symbols.is_empty() {
-                        // `import module`
-                        self.register_name(&part.module, SymbolRole::Assigned)?;
                     } else {
-                        // `from mymodule import myimport`
-                        for symbol in &part.symbols {
-                            if let Some(alias) = &symbol.alias {
-                                // `from mymodule import myimportname as myalias`
-                                self.register_name(alias, SymbolRole::Assigned)?;
-                            } else {
-                                self.register_name(&symbol.symbol, SymbolRole::Assigned)?;
-                            }
-                        }
+                        // `import module`
+                        self.register_name(&name.symbol, SymbolRole::Assigned)?;
                     }
                 }
             }

From de2e496e1ff6381db9938946cc257a2c7c31de3f Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Tue, 16 Jul 2019 22:04:38 +0200
Subject: [PATCH 025/459] Add initial draft of symtable module.

---
 src/lib.rs         | 2 +-
 src/symboltable.rs | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index 5e844b2e..de949be2 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -8,4 +8,4 @@ extern crate log;
 
 pub mod compile;
 pub mod error;
-mod symboltable;
+pub mod symboltable;
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 5490bd05..23301016 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -36,7 +36,7 @@ pub fn statements_to_symbol_table(
     Ok(symbol_table)
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum SymbolRole {
     Global,
     Nonlocal,
@@ -45,6 +45,7 @@ pub enum SymbolRole {
 }
 
 /// Captures all symbols in the current scope, and has a list of subscopes in this scope.
+#[derive(Clone)]
 pub struct SymbolScope {
     /// A set of symbols present on this scope level.
     pub symbols: HashMap<String, SymbolRole>,

From 5dc63b9e8ec261dc463cdd909ae4e6cf8822acfc Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Wed, 17 Jul 2019 09:36:11 +0200
Subject: [PATCH 026/459] Implement suggestion from clippy.

---
 src/symboltable.rs | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 23301016..3122af01 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -13,7 +13,7 @@ use rustpython_parser::location::Location;
 use std::collections::HashMap;
 
 pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolScope, SymbolTableError> {
-    let mut builder = SymbolTableBuilder::new();
+    let mut builder: SymbolTableBuilder = Default::default();
     builder.enter_scope();
     builder.scan_program(program)?;
     assert_eq!(builder.scopes.len(), 1);
@@ -26,7 +26,7 @@ pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolScope, SymbolTa
 pub fn statements_to_symbol_table(
     statements: &[ast::LocatedStatement],
 ) -> Result<SymbolScope, SymbolTableError> {
-    let mut builder = SymbolTableBuilder::new();
+    let mut builder: SymbolTableBuilder = Default::default();
     builder.enter_scope();
     builder.scan_statements(statements)?;
     assert_eq!(builder.scopes.len(), 1);
@@ -73,16 +73,18 @@ impl From<SymbolTableError> for CompileError {
 type SymbolTableResult = Result<(), SymbolTableError>;
 
 impl SymbolScope {
-    pub fn new() -> Self {
+    pub fn lookup(&self, name: &str) -> Option<&SymbolRole> {
+        self.symbols.get(name)
+    }
+}
+
+impl Default for SymbolScope {
+    fn default() -> Self {
         SymbolScope {
             symbols: HashMap::new(),
             sub_scopes: vec![],
         }
     }
-
-    pub fn lookup(&self, name: &str) -> Option<&SymbolRole> {
-        self.symbols.get(name)
-    }
 }
 
 impl std::fmt::Debug for SymbolScope {
@@ -153,13 +155,15 @@ pub struct SymbolTableBuilder {
     pub scopes: Vec<SymbolScope>,
 }
 
-impl SymbolTableBuilder {
-    pub fn new() -> Self {
+impl Default for SymbolTableBuilder {
+    fn default() -> Self {
         SymbolTableBuilder { scopes: vec![] }
     }
+}
 
+impl SymbolTableBuilder {
     pub fn enter_scope(&mut self) {
-        let scope = SymbolScope::new();
+        let scope = Default::default();
         self.scopes.push(scope);
     }
 

From 91b43d47fb476767193234da76a4e0e85f4b58ec Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Fri, 19 Jul 2019 17:31:57 +0200
Subject: [PATCH 027/459] Fix panic on import statement with levels but no
 module name.

---
 src/compile.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 8cd2ab35..8f3d695c 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -271,7 +271,7 @@ impl Compiler {
                 // import a, b, c as d
                 for name in names {
                     self.emit(Instruction::Import {
-                        name: name.symbol.clone(),
+                        name: Some(name.symbol.clone()),
                         symbols: vec![],
                         level: 0,
                     });
@@ -293,7 +293,7 @@ impl Compiler {
                 if import_star {
                     // from .... import *
                     self.emit(Instruction::ImportStar {
-                        name: module.clone().unwrap(),
+                        name: module.clone(),
                         level: *level,
                     });
                 } else {
@@ -303,7 +303,7 @@ impl Compiler {
 
                     // Load module once:
                     self.emit(Instruction::Import {
-                        name: module.clone().unwrap(),
+                        name: module.clone(),
                         symbols: from_list,
                         level: *level,
                     });

From eab3b357dd7715ada66d84e6c9943e0bb2da7be9 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Fri, 19 Jul 2019 22:05:35 +0200
Subject: [PATCH 028/459] Extend symtable module.

---
 Cargo.toml         | 1 +
 src/symboltable.rs | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 0a2c155c..6531fc91 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,6 +8,7 @@ license = "MIT"
 edition = "2018"
 
 [dependencies]
+indexmap = "1.0"
 rustpython-bytecode = { path = "../bytecode", version = "0.1.0" }
 rustpython-parser = { path = "../parser", version = "0.1.0" }
 num-complex = { version = "0.2", features = ["serde"] }
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 3122af01..9d4d6772 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -8,9 +8,9 @@ Inspirational file: https://github.com/python/cpython/blob/master/Python/symtabl
 */
 
 use crate::error::{CompileError, CompileErrorType};
+use indexmap::map::IndexMap;
 use rustpython_parser::ast;
 use rustpython_parser::location::Location;
-use std::collections::HashMap;
 
 pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolScope, SymbolTableError> {
     let mut builder: SymbolTableBuilder = Default::default();
@@ -48,7 +48,7 @@ pub enum SymbolRole {
 #[derive(Clone)]
 pub struct SymbolScope {
     /// A set of symbols present on this scope level.
-    pub symbols: HashMap<String, SymbolRole>,
+    pub symbols: IndexMap<String, SymbolRole>,
 
     /// A list of subscopes in the order as found in the
     /// AST nodes.
@@ -81,8 +81,8 @@ impl SymbolScope {
 impl Default for SymbolScope {
     fn default() -> Self {
         SymbolScope {
-            symbols: HashMap::new(),
-            sub_scopes: vec![],
+            symbols: Default::default(),
+            sub_scopes: Default::default(),
         }
     }
 }

From 9f7ef2050ec8b93b851d62982093a91d127f3882 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Sat, 20 Jul 2019 20:44:38 +0200
Subject: [PATCH 029/459] Add location to expressions. Change symboltable to
 use flags for symbols.

---
 src/compile.rs     | 196 ++++++++++++++++----------------
 src/symboltable.rs | 270 +++++++++++++++++++++++++--------------------
 2 files changed, 252 insertions(+), 214 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 8f3d695c..b06262c6 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -6,7 +6,7 @@
 //!   https://github.com/micropython/micropython/blob/master/py/compile.c
 
 use crate::error::{CompileError, CompileErrorType};
-use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolRole, SymbolScope};
+use crate::symboltable::{make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope};
 use num_complex::Complex64;
 use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Varargs};
 use rustpython_parser::{ast, parser};
@@ -75,7 +75,7 @@ pub fn compile_program(
 
 /// Compile a single Python expression to bytecode
 pub fn compile_statement_eval(
-    statement: Vec<ast::LocatedStatement>,
+    statement: Vec<ast::Statement>,
     source_path: String,
     optimize: u8,
 ) -> Result<CodeObject, CompileError> {
@@ -180,7 +180,7 @@ impl Compiler {
         for (i, statement) in program.statements.iter().enumerate() {
             let is_last = i == program.statements.len() - 1;
 
-            if let ast::Statement::Expression { ref expression } = statement.node {
+            if let ast::StatementType::Expression { ref expression } = statement.node {
                 self.compile_expression(expression)?;
 
                 if is_last {
@@ -209,12 +209,12 @@ impl Compiler {
     // Compile statement in eval mode:
     fn compile_statement_eval(
         &mut self,
-        statements: &[ast::LocatedStatement],
+        statements: &[ast::Statement],
         symbol_table: SymbolScope,
     ) -> Result<(), CompileError> {
         self.scope_stack.push(symbol_table);
         for statement in statements {
-            if let ast::Statement::Expression { ref expression } = statement.node {
+            if let ast::StatementType::Expression { ref expression } = statement.node {
                 self.compile_expression(expression)?;
             } else {
                 return Err(CompileError {
@@ -227,10 +227,7 @@ impl Compiler {
         Ok(())
     }
 
-    fn compile_statements(
-        &mut self,
-        statements: &[ast::LocatedStatement],
-    ) -> Result<(), CompileError> {
+    fn compile_statements(&mut self, statements: &[ast::Statement]) -> Result<(), CompileError> {
         for statement in statements {
             self.compile_statement(statement)?
         }
@@ -238,11 +235,13 @@ impl Compiler {
     }
 
     fn scope_for_name(&self, name: &str) -> bytecode::NameScope {
-        let role = self.lookup_name(name);
-        match role {
-            SymbolRole::Global => bytecode::NameScope::Global,
-            SymbolRole::Nonlocal => bytecode::NameScope::NonLocal,
-            _ => bytecode::NameScope::Local,
+        let symbol = self.lookup_name(name);
+        if symbol.is_global {
+            bytecode::NameScope::Global
+        } else if symbol.is_nonlocal {
+            bytecode::NameScope::NonLocal
+        } else {
+            bytecode::NameScope::Local
         }
     }
 
@@ -262,12 +261,13 @@ impl Compiler {
         });
     }
 
-    fn compile_statement(&mut self, statement: &ast::LocatedStatement) -> Result<(), CompileError> {
+    fn compile_statement(&mut self, statement: &ast::Statement) -> Result<(), CompileError> {
         trace!("Compiling {:?}", statement);
         self.set_source_location(&statement.location);
+        use ast::StatementType::*;
 
         match &statement.node {
-            ast::Statement::Import { names } => {
+            Import { names } => {
                 // import a, b, c as d
                 for name in names {
                     self.emit(Instruction::Import {
@@ -283,7 +283,7 @@ impl Compiler {
                     }
                 }
             }
-            ast::Statement::ImportFrom {
+            ImportFrom {
                 level,
                 module,
                 names,
@@ -326,16 +326,16 @@ impl Compiler {
                     self.emit(Instruction::Pop);
                 }
             }
-            ast::Statement::Expression { expression } => {
+            Expression { expression } => {
                 self.compile_expression(expression)?;
 
                 // Pop result of stack, since we not use it:
                 self.emit(Instruction::Pop);
             }
-            ast::Statement::Global { .. } | ast::Statement::Nonlocal { .. } => {
+            Global { .. } | Nonlocal { .. } => {
                 // Handled during symbol table construction.
             }
-            ast::Statement::If { test, body, orelse } => {
+            If { test, body, orelse } => {
                 let end_label = self.new_label();
                 match orelse {
                     None => {
@@ -358,7 +358,7 @@ impl Compiler {
                 }
                 self.set_label(end_label);
             }
-            ast::Statement::While { test, body, orelse } => {
+            While { test, body, orelse } => {
                 let start_label = self.new_label();
                 let else_label = self.new_label();
                 let end_label = self.new_label();
@@ -385,7 +385,7 @@ impl Compiler {
                 }
                 self.set_label(end_label);
             }
-            ast::Statement::With { items, body } => {
+            With { items, body } => {
                 let end_label = self.new_label();
                 for item in items {
                     self.compile_expression(&item.context_expr)?;
@@ -406,16 +406,16 @@ impl Compiler {
                 }
                 self.set_label(end_label);
             }
-            ast::Statement::For {
+            For {
                 target,
                 iter,
                 body,
                 orelse,
             } => self.compile_for(target, iter, body, orelse)?,
-            ast::Statement::AsyncFor { .. } => {
+            AsyncFor { .. } => {
                 unimplemented!("async for");
             }
-            ast::Statement::Raise { exception, cause } => match exception {
+            Raise { exception, cause } => match exception {
                 Some(value) => {
                     self.compile_expression(value)?;
                     match cause {
@@ -432,30 +432,30 @@ impl Compiler {
                     self.emit(Instruction::Raise { argc: 0 });
                 }
             },
-            ast::Statement::Try {
+            Try {
                 body,
                 handlers,
                 orelse,
                 finalbody,
             } => self.compile_try_statement(body, handlers, orelse, finalbody)?,
-            ast::Statement::FunctionDef {
+            FunctionDef {
                 name,
                 args,
                 body,
                 decorator_list,
                 returns,
             } => self.compile_function_def(name, args, body, decorator_list, returns)?,
-            ast::Statement::AsyncFunctionDef { .. } => {
+            AsyncFunctionDef { .. } => {
                 unimplemented!("async def");
             }
-            ast::Statement::ClassDef {
+            ClassDef {
                 name,
                 body,
                 bases,
                 keywords,
                 decorator_list,
             } => self.compile_class_def(name, body, bases, keywords, decorator_list)?,
-            ast::Statement::Assert { test, msg } => {
+            Assert { test, msg } => {
                 // if some flag, ignore all assert statements!
                 if self.optimize == 0 {
                     let end_label = self.new_label();
@@ -481,7 +481,7 @@ impl Compiler {
                     self.set_label(end_label);
                 }
             }
-            ast::Statement::Break => {
+            Break => {
                 if !self.in_loop {
                     return Err(CompileError {
                         error: CompileErrorType::InvalidBreak,
@@ -490,7 +490,7 @@ impl Compiler {
                 }
                 self.emit(Instruction::Break);
             }
-            ast::Statement::Continue => {
+            Continue => {
                 if !self.in_loop {
                     return Err(CompileError {
                         error: CompileErrorType::InvalidContinue,
@@ -499,7 +499,7 @@ impl Compiler {
                 }
                 self.emit(Instruction::Continue);
             }
-            ast::Statement::Return { value } => {
+            Return { value } => {
                 if !self.in_function_def {
                     return Err(CompileError {
                         error: CompileErrorType::InvalidReturn,
@@ -519,7 +519,7 @@ impl Compiler {
 
                 self.emit(Instruction::ReturnValue);
             }
-            ast::Statement::Assign { targets, value } => {
+            Assign { targets, value } => {
                 self.compile_expression(value)?;
 
                 for (i, target) in targets.iter().enumerate() {
@@ -529,7 +529,7 @@ impl Compiler {
                     self.compile_store(target)?;
                 }
             }
-            ast::Statement::AugAssign { target, op, value } => {
+            AugAssign { target, op, value } => {
                 self.compile_expression(target)?;
                 self.compile_expression(value)?;
 
@@ -537,12 +537,12 @@ impl Compiler {
                 self.compile_op(op, true);
                 self.compile_store(target)?;
             }
-            ast::Statement::Delete { targets } => {
+            Delete { targets } => {
                 for target in targets {
                     self.compile_delete(target)?;
                 }
             }
-            ast::Statement::Pass => {
+            Pass => {
                 self.emit(Instruction::Pass);
             }
         }
@@ -550,24 +550,24 @@ impl Compiler {
     }
 
     fn compile_delete(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
-        match expression {
-            ast::Expression::Identifier { name } => {
+        match &expression.node {
+            ast::ExpressionType::Identifier { name } => {
                 self.emit(Instruction::DeleteName {
                     name: name.to_string(),
                 });
             }
-            ast::Expression::Attribute { value, name } => {
+            ast::ExpressionType::Attribute { value, name } => {
                 self.compile_expression(value)?;
                 self.emit(Instruction::DeleteAttr {
                     name: name.to_string(),
                 });
             }
-            ast::Expression::Subscript { a, b } => {
+            ast::ExpressionType::Subscript { a, b } => {
                 self.compile_expression(a)?;
                 self.compile_expression(b)?;
                 self.emit(Instruction::DeleteSubscript);
             }
-            ast::Expression::Tuple { elements } => {
+            ast::ExpressionType::Tuple { elements } => {
                 for element in elements {
                     self.compile_delete(element)?;
                 }
@@ -663,10 +663,10 @@ impl Compiler {
 
     fn compile_try_statement(
         &mut self,
-        body: &[ast::LocatedStatement],
+        body: &[ast::Statement],
         handlers: &[ast::ExceptHandler],
-        orelse: &Option<Vec<ast::LocatedStatement>>,
-        finalbody: &Option<Vec<ast::LocatedStatement>>,
+        orelse: &Option<Vec<ast::Statement>>,
+        finalbody: &Option<Vec<ast::Statement>>,
     ) -> Result<(), CompileError> {
         let mut handler_label = self.new_label();
         let finally_label = self.new_label();
@@ -764,7 +764,7 @@ impl Compiler {
         &mut self,
         name: &str,
         args: &ast::Parameters,
-        body: &[ast::LocatedStatement],
+        body: &[ast::Statement],
         decorator_list: &[ast::Expression],
         returns: &Option<ast::Expression>, // TODO: use type hint somehow..
     ) -> Result<(), CompileError> {
@@ -858,7 +858,7 @@ impl Compiler {
     fn compile_class_def(
         &mut self,
         name: &str,
-        body: &[ast::LocatedStatement],
+        body: &[ast::Statement],
         bases: &[ast::Expression],
         keywords: &[ast::Keyword],
         decorator_list: &[ast::Expression],
@@ -989,8 +989,8 @@ impl Compiler {
         &mut self,
         target: &ast::Expression,
         iter: &ast::Expression,
-        body: &[ast::LocatedStatement],
-        orelse: &Option<Vec<ast::LocatedStatement>>,
+        body: &[ast::Statement],
+        orelse: &Option<Vec<ast::Statement>>,
     ) -> Result<(), CompileError> {
         // Start loop
         let start_label = self.new_label();
@@ -1104,27 +1104,27 @@ impl Compiler {
     }
 
     fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> {
-        match target {
-            ast::Expression::Identifier { name } => {
+        match &target.node {
+            ast::ExpressionType::Identifier { name } => {
                 self.store_name(name);
             }
-            ast::Expression::Subscript { a, b } => {
+            ast::ExpressionType::Subscript { a, b } => {
                 self.compile_expression(a)?;
                 self.compile_expression(b)?;
                 self.emit(Instruction::StoreSubscript);
             }
-            ast::Expression::Attribute { value, name } => {
+            ast::ExpressionType::Attribute { value, name } => {
                 self.compile_expression(value)?;
                 self.emit(Instruction::StoreAttr {
                     name: name.to_string(),
                 });
             }
-            ast::Expression::List { elements } | ast::Expression::Tuple { elements } => {
+            ast::ExpressionType::List { elements } | ast::ExpressionType::Tuple { elements } => {
                 let mut seen_star = false;
 
                 // Scan for star args:
                 for (i, element) in elements.iter().enumerate() {
-                    if let ast::Expression::Starred { .. } = element {
+                    if let ast::ExpressionType::Starred { .. } = &element.node {
                         if seen_star {
                             return Err(CompileError {
                                 error: CompileErrorType::StarArgs,
@@ -1147,7 +1147,7 @@ impl Compiler {
                 }
 
                 for element in elements {
-                    if let ast::Expression::Starred { value } = element {
+                    if let ast::ExpressionType::Starred { value } = &element.node {
                         self.compile_store(value)?;
                     } else {
                         self.compile_store(element)?;
@@ -1192,8 +1192,8 @@ impl Compiler {
         context: EvalContext,
     ) -> Result<(), CompileError> {
         // Compile expression for test, and jump to label if false
-        match expression {
-            ast::Expression::BoolOp { a, op, b } => match op {
+        match &expression.node {
+            ast::ExpressionType::BoolOp { a, op, b } => match op {
                 ast::BooleanOperator::And => {
                     let f = false_label.unwrap_or_else(|| self.new_label());
                     self.compile_test(a, None, Some(f), context)?;
@@ -1246,23 +1246,27 @@ impl Compiler {
 
     fn compile_expression(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
         trace!("Compiling {:?}", expression);
-        match expression {
-            ast::Expression::Call {
+        use ast::ExpressionType::*;
+        match &expression.node {
+            Call {
                 function,
                 args,
                 keywords,
             } => self.compile_call(function, args, keywords)?,
-            ast::Expression::BoolOp { .. } => {
-                self.compile_test(expression, None, None, EvalContext::Expression)?
-            }
-            ast::Expression::Binop { a, op, b } => {
+            BoolOp { .. } => self.compile_test(
+                expression,
+                Option::None,
+                Option::None,
+                EvalContext::Expression,
+            )?,
+            Binop { a, op, b } => {
                 self.compile_expression(a)?;
                 self.compile_expression(b)?;
 
                 // Perform operation:
                 self.compile_op(op, false);
             }
-            ast::Expression::Subscript { a, b } => {
+            Subscript { a, b } => {
                 self.compile_expression(a)?;
                 self.compile_expression(b)?;
                 self.emit(Instruction::BinaryOperation {
@@ -1270,7 +1274,7 @@ impl Compiler {
                     inplace: false,
                 });
             }
-            ast::Expression::Unop { op, a } => {
+            Unop { op, a } => {
                 self.compile_expression(a)?;
 
                 // Perform operation:
@@ -1283,16 +1287,16 @@ impl Compiler {
                 let i = Instruction::UnaryOperation { op: i };
                 self.emit(i);
             }
-            ast::Expression::Attribute { value, name } => {
+            Attribute { value, name } => {
                 self.compile_expression(value)?;
                 self.emit(Instruction::LoadAttr {
                     name: name.to_string(),
                 });
             }
-            ast::Expression::Compare { vals, ops } => {
+            Compare { vals, ops } => {
                 self.compile_chained_comparison(vals, ops)?;
             }
-            ast::Expression::Number { value } => {
+            Number { value } => {
                 let const_value = match value {
                     ast::Number::Integer { value } => bytecode::Constant::Integer {
                         value: value.clone(),
@@ -1304,7 +1308,7 @@ impl Compiler {
                 };
                 self.emit(Instruction::LoadConst { value: const_value });
             }
-            ast::Expression::List { elements } => {
+            List { elements } => {
                 let size = elements.len();
                 let must_unpack = self.gather_elements(elements)?;
                 self.emit(Instruction::BuildList {
@@ -1312,7 +1316,7 @@ impl Compiler {
                     unpack: must_unpack,
                 });
             }
-            ast::Expression::Tuple { elements } => {
+            Tuple { elements } => {
                 let size = elements.len();
                 let must_unpack = self.gather_elements(elements)?;
                 self.emit(Instruction::BuildTuple {
@@ -1320,7 +1324,7 @@ impl Compiler {
                     unpack: must_unpack,
                 });
             }
-            ast::Expression::Set { elements } => {
+            Set { elements } => {
                 let size = elements.len();
                 let must_unpack = self.gather_elements(elements)?;
                 self.emit(Instruction::BuildSet {
@@ -1328,7 +1332,7 @@ impl Compiler {
                     unpack: must_unpack,
                 });
             }
-            ast::Expression::Dict { elements } => {
+            Dict { elements } => {
                 let size = elements.len();
                 let has_double_star = elements.iter().any(|e| e.0.is_none());
                 for (key, value) in elements {
@@ -1351,14 +1355,14 @@ impl Compiler {
                     unpack: has_double_star,
                 });
             }
-            ast::Expression::Slice { elements } => {
+            Slice { elements } => {
                 let size = elements.len();
                 for element in elements {
                     self.compile_expression(element)?;
                 }
                 self.emit(Instruction::BuildSlice { size });
             }
-            ast::Expression::Yield { value } => {
+            Yield { value } => {
                 if !self.in_function_def {
                     return Err(CompileError {
                         error: CompileErrorType::InvalidYield,
@@ -1368,16 +1372,16 @@ impl Compiler {
                 self.mark_generator();
                 match value {
                     Some(expression) => self.compile_expression(expression)?,
-                    None => self.emit(Instruction::LoadConst {
+                    Option::None => self.emit(Instruction::LoadConst {
                         value: bytecode::Constant::None,
                     }),
                 };
                 self.emit(Instruction::YieldValue);
             }
-            ast::Expression::Await { .. } => {
+            Await { .. } => {
                 unimplemented!("await");
             }
-            ast::Expression::YieldFrom { value } => {
+            YieldFrom { value } => {
                 self.mark_generator();
                 self.compile_expression(value)?;
                 self.emit(Instruction::GetIter);
@@ -1386,40 +1390,40 @@ impl Compiler {
                 });
                 self.emit(Instruction::YieldFrom);
             }
-            ast::Expression::True => {
+            True => {
                 self.emit(Instruction::LoadConst {
                     value: bytecode::Constant::Boolean { value: true },
                 });
             }
-            ast::Expression::False => {
+            False => {
                 self.emit(Instruction::LoadConst {
                     value: bytecode::Constant::Boolean { value: false },
                 });
             }
-            ast::Expression::None => {
+            None => {
                 self.emit(Instruction::LoadConst {
                     value: bytecode::Constant::None,
                 });
             }
-            ast::Expression::Ellipsis => {
+            Ellipsis => {
                 self.emit(Instruction::LoadConst {
                     value: bytecode::Constant::Ellipsis,
                 });
             }
-            ast::Expression::String { value } => {
+            String { value } => {
                 self.compile_string(value)?;
             }
-            ast::Expression::Bytes { value } => {
+            Bytes { value } => {
                 self.emit(Instruction::LoadConst {
                     value: bytecode::Constant::Bytes {
                         value: value.clone(),
                     },
                 });
             }
-            ast::Expression::Identifier { name } => {
+            Identifier { name } => {
                 self.load_name(name);
             }
-            ast::Expression::Lambda { args, body } => {
+            Lambda { args, body } => {
                 let name = "<lambda>".to_string();
                 // no need to worry about the self.loop_depth because there are no loops in lambda expressions
                 let flags = self.enter_function(&name, args)?;
@@ -1438,18 +1442,18 @@ impl Compiler {
                 // Turn code object into function object:
                 self.emit(Instruction::MakeFunction { flags });
             }
-            ast::Expression::Comprehension { kind, generators } => {
+            Comprehension { kind, generators } => {
                 self.compile_comprehension(kind, generators)?;
             }
-            ast::Expression::Starred { value } => {
+            Starred { value } => {
                 self.compile_expression(value)?;
                 self.emit(Instruction::Unpack);
                 panic!("We should not just unpack a starred args, since the size is unknown.");
             }
-            ast::Expression::IfExpression { test, body, orelse } => {
+            IfExpression { test, body, orelse } => {
                 let no_label = self.new_label();
                 let end_label = self.new_label();
-                self.compile_test(test, None, None, EvalContext::Expression)?;
+                self.compile_test(test, Option::None, Option::None, EvalContext::Expression)?;
                 self.emit(Instruction::JumpIfFalse { target: no_label });
                 // True case
                 self.compile_expression(body)?;
@@ -1557,7 +1561,7 @@ impl Compiler {
     fn gather_elements(&mut self, elements: &[ast::Expression]) -> Result<bool, CompileError> {
         // First determine if we have starred elements:
         let has_stars = elements.iter().any(|e| {
-            if let ast::Expression::Starred { .. } = e {
+            if let ast::ExpressionType::Starred { .. } = &e.node {
                 true
             } else {
                 false
@@ -1565,7 +1569,7 @@ impl Compiler {
         });
 
         for element in elements {
-            if let ast::Expression::Starred { value } = element {
+            if let ast::ExpressionType::Starred { value } = &element.node {
                 self.compile_expression(value)?;
             } else {
                 self.compile_expression(element)?;
@@ -1792,7 +1796,7 @@ impl Compiler {
         assert!(scope.sub_scopes.is_empty());
     }
 
-    fn lookup_name(&self, name: &str) -> &SymbolRole {
+    fn lookup_name(&self, name: &str) -> &Symbol {
         // println!("Looking up {:?}", name);
         let scope = self.scope_stack.last().unwrap();
         scope.lookup(name).unwrap()
@@ -1846,10 +1850,10 @@ impl Compiler {
     }
 }
 
-fn get_doc(body: &[ast::LocatedStatement]) -> (&[ast::LocatedStatement], Option<String>) {
+fn get_doc(body: &[ast::Statement]) -> (&[ast::Statement], Option<String>) {
     if let Some(val) = body.get(0) {
-        if let ast::Statement::Expression { ref expression } = val.node {
-            if let ast::Expression::String { ref value } = expression {
+        if let ast::StatementType::Expression { ref expression } = val.node {
+            if let ast::ExpressionType::String { value } = &expression.node {
                 if let ast::StringGroup::Constant { ref value } = value {
                     if let Some((_, body_rest)) = body.split_first() {
                         return (body_rest, Some(value.to_string()));
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 9d4d6772..d80755d3 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -24,7 +24,7 @@ pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolScope, SymbolTa
 }
 
 pub fn statements_to_symbol_table(
-    statements: &[ast::LocatedStatement],
+    statements: &[ast::Statement],
 ) -> Result<SymbolScope, SymbolTableError> {
     let mut builder: SymbolTableBuilder = Default::default();
     builder.enter_scope();
@@ -36,25 +36,46 @@ pub fn statements_to_symbol_table(
     Ok(symbol_table)
 }
 
-#[derive(Debug, Clone)]
-pub enum SymbolRole {
-    Global,
-    Nonlocal,
-    Used,
-    Assigned,
-}
-
 /// Captures all symbols in the current scope, and has a list of subscopes in this scope.
-#[derive(Clone)]
+#[derive(Clone, Default)]
 pub struct SymbolScope {
     /// A set of symbols present on this scope level.
-    pub symbols: IndexMap<String, SymbolRole>,
+    pub symbols: IndexMap<String, Symbol>,
 
     /// A list of subscopes in the order as found in the
     /// AST nodes.
     pub sub_scopes: Vec<SymbolScope>,
 }
 
+#[derive(Debug, Clone)]
+pub struct Symbol {
+    pub name: String,
+    pub is_global: bool,
+    pub is_local: bool,
+    pub is_nonlocal: bool,
+    pub is_param: bool,
+    pub is_referenced: bool,
+    pub is_assigned: bool,
+    pub is_parameter: bool,
+    pub is_free: bool,
+}
+
+impl Symbol {
+    fn new(name: &str) -> Self {
+        Symbol {
+            name: name.to_string(),
+            is_global: false,
+            is_local: false,
+            is_nonlocal: false,
+            is_param: false,
+            is_referenced: false,
+            is_assigned: false,
+            is_parameter: false,
+            is_free: false,
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct SymbolTableError {
     error: String,
@@ -73,20 +94,11 @@ impl From<SymbolTableError> for CompileError {
 type SymbolTableResult = Result<(), SymbolTableError>;
 
 impl SymbolScope {
-    pub fn lookup(&self, name: &str) -> Option<&SymbolRole> {
+    pub fn lookup(&self, name: &str) -> Option<&Symbol> {
         self.symbols.get(name)
     }
 }
 
-impl Default for SymbolScope {
-    fn default() -> Self {
-        SymbolScope {
-            symbols: Default::default(),
-            sub_scopes: Default::default(),
-        }
-    }
-}
-
 impl std::fmt::Debug for SymbolScope {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(
@@ -111,74 +123,73 @@ fn analyze_symbol_table(
     }
 
     // Analyze symbols:
-    for (symbol_name, symbol_role) in &symbol_scope.symbols {
-        analyze_symbol(symbol_name, symbol_role, parent_symbol_scope)?;
+    for symbol in symbol_scope.symbols.values() {
+        analyze_symbol(symbol, parent_symbol_scope)?;
     }
 
     Ok(())
 }
 
-#[allow(clippy::single_match)]
-fn analyze_symbol(
-    symbol_name: &str,
-    symbol_role: &SymbolRole,
-    parent_symbol_scope: Option<&SymbolScope>,
-) -> SymbolTableResult {
-    match symbol_role {
-        SymbolRole::Nonlocal => {
-            // check if name is defined in parent scope!
-            if let Some(parent_symbol_scope) = parent_symbol_scope {
-                if !parent_symbol_scope.symbols.contains_key(symbol_name) {
-                    return Err(SymbolTableError {
-                        error: format!("no binding for nonlocal '{}' found", symbol_name),
-                        location: Default::default(),
-                    });
-                }
-            } else {
+fn analyze_symbol(symbol: &Symbol, parent_symbol_scope: Option<&SymbolScope>) -> SymbolTableResult {
+    if symbol.is_nonlocal {
+        // check if name is defined in parent scope!
+        if let Some(parent_symbol_scope) = parent_symbol_scope {
+            if !parent_symbol_scope.symbols.contains_key(&symbol.name) {
                 return Err(SymbolTableError {
-                    error: format!(
-                        "nonlocal {} defined at place without an enclosing scope",
-                        symbol_name
-                    ),
+                    error: format!("no binding for nonlocal '{}' found", symbol.name),
                     location: Default::default(),
                 });
             }
+        } else {
+            return Err(SymbolTableError {
+                error: format!(
+                    "nonlocal {} defined at place without an enclosing scope",
+                    symbol.name
+                ),
+                location: Default::default(),
+            });
         }
-        // TODO: add more checks for globals
-        _ => {}
     }
+
+    // TODO: add more checks for globals
+
     Ok(())
 }
 
-pub struct SymbolTableBuilder {
-    // Scope stack.
-    pub scopes: Vec<SymbolScope>,
+#[derive(Debug, Clone)]
+enum SymbolRole {
+    Global,
+    Nonlocal,
+    Used,
+    Assigned,
 }
 
-impl Default for SymbolTableBuilder {
-    fn default() -> Self {
-        SymbolTableBuilder { scopes: vec![] }
-    }
+#[derive(Default)]
+struct SymbolTableBuilder {
+    // Scope stack.
+    scopes: Vec<SymbolScope>,
 }
 
 impl SymbolTableBuilder {
-    pub fn enter_scope(&mut self) {
+    fn enter_scope(&mut self) {
         let scope = Default::default();
         self.scopes.push(scope);
+        // self.work_scopes.push(Default::default());
     }
 
     fn leave_scope(&mut self) {
         // Pop scope and add to subscopes of parent scope.
+        // let work_scope = self.work_scopes.pop().unwrap();
         let scope = self.scopes.pop().unwrap();
         self.scopes.last_mut().unwrap().sub_scopes.push(scope);
     }
 
-    pub fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult {
+    fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult {
         self.scan_statements(&program.statements)?;
         Ok(())
     }
 
-    pub fn scan_statements(&mut self, statements: &[ast::LocatedStatement]) -> SymbolTableResult {
+    fn scan_statements(&mut self, statements: &[ast::Statement]) -> SymbolTableResult {
         for statement in statements {
             self.scan_statement(statement)?;
         }
@@ -210,26 +221,27 @@ impl SymbolTableBuilder {
         Ok(())
     }
 
-    fn scan_statement(&mut self, statement: &ast::LocatedStatement) -> SymbolTableResult {
+    fn scan_statement(&mut self, statement: &ast::Statement) -> SymbolTableResult {
+        use ast::StatementType::*;
         match &statement.node {
-            ast::Statement::Global { names } => {
+            Global { names } => {
                 for name in names {
                     self.register_name(name, SymbolRole::Global)?;
                 }
             }
-            ast::Statement::Nonlocal { names } => {
+            Nonlocal { names } => {
                 for name in names {
                     self.register_name(name, SymbolRole::Nonlocal)?;
                 }
             }
-            ast::Statement::FunctionDef {
+            FunctionDef {
                 name,
                 body,
                 args,
                 decorator_list,
                 returns,
             }
-            | ast::Statement::AsyncFunctionDef {
+            | AsyncFunctionDef {
                 name,
                 body,
                 args,
@@ -247,7 +259,7 @@ impl SymbolTableBuilder {
                 }
                 self.leave_scope();
             }
-            ast::Statement::ClassDef {
+            ClassDef {
                 name,
                 body,
                 bases,
@@ -264,21 +276,21 @@ impl SymbolTableBuilder {
                 }
                 self.scan_expressions(decorator_list)?;
             }
-            ast::Statement::Expression { expression } => self.scan_expression(expression)?,
-            ast::Statement::If { test, body, orelse } => {
+            Expression { expression } => self.scan_expression(expression)?,
+            If { test, body, orelse } => {
                 self.scan_expression(test)?;
                 self.scan_statements(body)?;
                 if let Some(code) = orelse {
                     self.scan_statements(code)?;
                 }
             }
-            ast::Statement::For {
+            For {
                 target,
                 iter,
                 body,
                 orelse,
             }
-            | ast::Statement::AsyncFor {
+            | AsyncFor {
                 target,
                 iter,
                 body,
@@ -291,17 +303,17 @@ impl SymbolTableBuilder {
                     self.scan_statements(code)?;
                 }
             }
-            ast::Statement::While { test, body, orelse } => {
+            While { test, body, orelse } => {
                 self.scan_expression(test)?;
                 self.scan_statements(body)?;
                 if let Some(code) = orelse {
                     self.scan_statements(code)?;
                 }
             }
-            ast::Statement::Break | ast::Statement::Continue | ast::Statement::Pass => {
+            Break | Continue | Pass => {
                 // No symbols here.
             }
-            ast::Statement::Import { names } | ast::Statement::ImportFrom { names, .. } => {
+            Import { names } | ImportFrom { names, .. } => {
                 for name in names {
                     if let Some(alias) = &name.alias {
                         // `import mymodule as myalias`
@@ -312,29 +324,29 @@ impl SymbolTableBuilder {
                     }
                 }
             }
-            ast::Statement::Return { value } => {
+            Return { value } => {
                 if let Some(expression) = value {
                     self.scan_expression(expression)?;
                 }
             }
-            ast::Statement::Assert { test, msg } => {
+            Assert { test, msg } => {
                 self.scan_expression(test)?;
                 if let Some(expression) = msg {
                     self.scan_expression(expression)?;
                 }
             }
-            ast::Statement::Delete { targets } => {
+            Delete { targets } => {
                 self.scan_expressions(targets)?;
             }
-            ast::Statement::Assign { targets, value } => {
+            Assign { targets, value } => {
                 self.scan_expressions(targets)?;
                 self.scan_expression(value)?;
             }
-            ast::Statement::AugAssign { target, value, .. } => {
+            AugAssign { target, value, .. } => {
                 self.scan_expression(target)?;
                 self.scan_expression(value)?;
             }
-            ast::Statement::With { items, body } => {
+            With { items, body } => {
                 for item in items {
                     self.scan_expression(&item.context_expr)?;
                     if let Some(expression) = &item.optional_vars {
@@ -343,7 +355,7 @@ impl SymbolTableBuilder {
                 }
                 self.scan_statements(body)?;
             }
-            ast::Statement::Try {
+            Try {
                 body,
                 handlers,
                 orelse,
@@ -366,7 +378,7 @@ impl SymbolTableBuilder {
                     self.scan_statements(code)?;
                 }
             }
-            ast::Statement::Raise { exception, cause } => {
+            Raise { exception, cause } => {
                 if let Some(expression) = exception {
                     self.scan_expression(expression)?;
                 }
@@ -386,26 +398,27 @@ impl SymbolTableBuilder {
     }
 
     fn scan_expression(&mut self, expression: &ast::Expression) -> SymbolTableResult {
-        match expression {
-            ast::Expression::Binop { a, b, .. } => {
+        use ast::ExpressionType::*;
+        match &expression.node {
+            Binop { a, b, .. } => {
                 self.scan_expression(a)?;
                 self.scan_expression(b)?;
             }
-            ast::Expression::BoolOp { a, b, .. } => {
+            BoolOp { a, b, .. } => {
                 self.scan_expression(a)?;
                 self.scan_expression(b)?;
             }
-            ast::Expression::Compare { vals, .. } => {
+            Compare { vals, .. } => {
                 self.scan_expressions(vals)?;
             }
-            ast::Expression::Subscript { a, b } => {
+            Subscript { a, b } => {
                 self.scan_expression(a)?;
                 self.scan_expression(b)?;
             }
-            ast::Expression::Attribute { value, .. } => {
+            Attribute { value, .. } => {
                 self.scan_expression(value)?;
             }
-            ast::Expression::Dict { elements } => {
+            Dict { elements } => {
                 for (key, value) in elements {
                     if let Some(key) = key {
                         self.scan_expression(key)?;
@@ -415,36 +428,30 @@ impl SymbolTableBuilder {
                     self.scan_expression(value)?;
                 }
             }
-            ast::Expression::Await { value } => {
+            Await { value } => {
                 self.scan_expression(value)?;
             }
-            ast::Expression::Yield { value } => {
+            Yield { value } => {
                 if let Some(expression) = value {
                     self.scan_expression(expression)?;
                 }
             }
-            ast::Expression::YieldFrom { value } => {
+            YieldFrom { value } => {
                 self.scan_expression(value)?;
             }
-            ast::Expression::Unop { a, .. } => {
+            Unop { a, .. } => {
                 self.scan_expression(a)?;
             }
-            ast::Expression::True
-            | ast::Expression::False
-            | ast::Expression::None
-            | ast::Expression::Ellipsis => {}
-            ast::Expression::Number { .. } => {}
-            ast::Expression::Starred { value } => {
+            True | False | None | Ellipsis => {}
+            Number { .. } => {}
+            Starred { value } => {
                 self.scan_expression(value)?;
             }
-            ast::Expression::Bytes { .. } => {}
-            ast::Expression::Tuple { elements }
-            | ast::Expression::Set { elements }
-            | ast::Expression::List { elements }
-            | ast::Expression::Slice { elements } => {
+            Bytes { .. } => {}
+            Tuple { elements } | Set { elements } | List { elements } | Slice { elements } => {
                 self.scan_expressions(elements)?;
             }
-            ast::Expression::Comprehension { kind, generators } => {
+            Comprehension { kind, generators } => {
                 match **kind {
                     ast::ComprehensionKind::GeneratorExpression { ref element }
                     | ast::ComprehensionKind::List { ref element }
@@ -465,7 +472,7 @@ impl SymbolTableBuilder {
                     }
                 }
             }
-            ast::Expression::Call {
+            Call {
                 function,
                 args,
                 keywords,
@@ -476,18 +483,18 @@ impl SymbolTableBuilder {
                     self.scan_expression(&keyword.value)?;
                 }
             }
-            ast::Expression::String { value } => {
+            String { value } => {
                 self.scan_string_group(value)?;
             }
-            ast::Expression::Identifier { name } => {
+            Identifier { name } => {
                 self.register_name(name, SymbolRole::Used)?;
             }
-            ast::Expression::Lambda { args, body } => {
+            Lambda { args, body } => {
                 self.enter_function(args)?;
                 self.scan_expression(body)?;
                 self.leave_scope();
             }
-            ast::Expression::IfExpression { test, body, orelse } => {
+            IfExpression { test, body, orelse } => {
                 self.scan_expression(test)?;
                 self.scan_expression(body)?;
                 self.scan_expression(orelse)?;
@@ -549,6 +556,8 @@ impl SymbolTableBuilder {
         let scope_depth = self.scopes.len();
         let current_scope = self.scopes.last_mut().unwrap();
         let location = Default::default();
+
+        // Some checks:
         if current_scope.symbols.contains_key(name) {
             // Role already set..
             match role {
@@ -568,22 +577,47 @@ impl SymbolTableBuilder {
                     // Ok?
                 }
             }
-        } else {
-            match role {
-                SymbolRole::Nonlocal => {
-                    if scope_depth < 2 {
-                        return Err(SymbolTableError {
-                            error: format!("cannot define nonlocal '{}' at top level.", name),
-                            location,
-                        });
-                    }
-                }
-                _ => {
-                    // Ok!
+        }
+
+        // Some more checks:
+        match role {
+            SymbolRole::Nonlocal => {
+                if scope_depth < 2 {
+                    return Err(SymbolTableError {
+                        error: format!("cannot define nonlocal '{}' at top level.", name),
+                        location,
+                    });
                 }
             }
-            current_scope.symbols.insert(name.to_string(), role);
+            _ => {
+                // Ok!
+            }
+        }
+
+        // Insert symbol when required:
+        if !current_scope.symbols.contains_key(name) {
+            let symbol = Symbol::new(name);
+            current_scope.symbols.insert(name.to_string(), symbol);
+        }
+
+        // Set proper flags on symbol:
+        let symbol = current_scope.symbols.get_mut(name).unwrap();
+        match role {
+            SymbolRole::Nonlocal => {
+                symbol.is_nonlocal = true;
+            }
+            SymbolRole::Assigned => {
+                symbol.is_assigned = true;
+                // symbol.is_local = true;
+            }
+            SymbolRole::Global => {
+                symbol.is_global = true;
+            }
+            SymbolRole::Used => {
+                symbol.is_referenced = true;
+            }
         }
+
         Ok(())
     }
 }

From 7e9b3ddc1f9bb49b94f6bbf3d8d24140a3538efd Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Thu, 25 Jul 2019 23:54:31 +0200
Subject: [PATCH 030/459] Implement review comments and improve parsing a bit.

---
 src/compile.rs     | 60 +++++++++++++++++++++++++++++-----------------
 src/symboltable.rs | 17 +++----------
 2 files changed, 41 insertions(+), 36 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index b06262c6..3fa6bdbe 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -385,35 +385,47 @@ impl Compiler {
                 }
                 self.set_label(end_label);
             }
-            With { items, body } => {
-                let end_label = self.new_label();
-                for item in items {
-                    self.compile_expression(&item.context_expr)?;
-                    self.emit(Instruction::SetupWith { end: end_label });
-                    match &item.optional_vars {
-                        Some(var) => {
-                            self.compile_store(var)?;
-                        }
-                        None => {
-                            self.emit(Instruction::Pop);
+            With {
+                is_async,
+                items,
+                body,
+            } => {
+                if *is_async {
+                    unimplemented!("async with");
+                } else {
+                    let end_label = self.new_label();
+                    for item in items {
+                        self.compile_expression(&item.context_expr)?;
+                        self.emit(Instruction::SetupWith { end: end_label });
+                        match &item.optional_vars {
+                            Some(var) => {
+                                self.compile_store(var)?;
+                            }
+                            None => {
+                                self.emit(Instruction::Pop);
+                            }
                         }
                     }
-                }
 
-                self.compile_statements(body)?;
-                for _ in 0..items.len() {
-                    self.emit(Instruction::CleanupWith { end: end_label });
+                    self.compile_statements(body)?;
+                    for _ in 0..items.len() {
+                        self.emit(Instruction::CleanupWith { end: end_label });
+                    }
+                    self.set_label(end_label);
                 }
-                self.set_label(end_label);
             }
             For {
+                is_async,
                 target,
                 iter,
                 body,
                 orelse,
-            } => self.compile_for(target, iter, body, orelse)?,
-            AsyncFor { .. } => {
-                unimplemented!("async for");
+            } => {
+                if *is_async {
+                    unimplemented!("async for");
+                } else {
+                    self.compile_for(target, iter, body, orelse)?
+                }
             }
             Raise { exception, cause } => match exception {
                 Some(value) => {
@@ -439,14 +451,18 @@ impl Compiler {
                 finalbody,
             } => self.compile_try_statement(body, handlers, orelse, finalbody)?,
             FunctionDef {
+                is_async,
                 name,
                 args,
                 body,
                 decorator_list,
                 returns,
-            } => self.compile_function_def(name, args, body, decorator_list, returns)?,
-            AsyncFunctionDef { .. } => {
-                unimplemented!("async def");
+            } => {
+                if *is_async {
+                    unimplemented!("async def");
+                } else {
+                    self.compile_function_def(name, args, body, decorator_list, returns)?
+                }
             }
             ClassDef {
                 name,
diff --git a/src/symboltable.rs b/src/symboltable.rs
index d80755d3..b12e0f92 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -240,13 +240,7 @@ impl SymbolTableBuilder {
                 args,
                 decorator_list,
                 returns,
-            }
-            | AsyncFunctionDef {
-                name,
-                body,
-                args,
-                decorator_list,
-                returns,
+                ..
             } => {
                 self.scan_expressions(decorator_list)?;
                 self.register_name(name, SymbolRole::Assigned)?;
@@ -289,12 +283,7 @@ impl SymbolTableBuilder {
                 iter,
                 body,
                 orelse,
-            }
-            | AsyncFor {
-                target,
-                iter,
-                body,
-                orelse,
+                ..
             } => {
                 self.scan_expression(target)?;
                 self.scan_expression(iter)?;
@@ -346,7 +335,7 @@ impl SymbolTableBuilder {
                 self.scan_expression(target)?;
                 self.scan_expression(value)?;
             }
-            With { items, body } => {
+            With { items, body, .. } => {
                 for item in items {
                     self.scan_expression(&item.context_expr)?;
                     if let Some(expression) = &item.optional_vars {

From 8d1da0920dca819b815325d49f173bc1cee9655b Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 2 Aug 2019 23:53:46 -0500
Subject: [PATCH 031/459] Add basic constant optimization to the compiler

---
 src/compile.rs | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/src/compile.rs b/src/compile.rs
index 3fa6bdbe..2b7d34e7 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1481,6 +1481,7 @@ impl Compiler {
                 self.set_label(end_label);
             }
         }
+        self.optimize_instruction();
         Ok(())
     }
 
@@ -1798,6 +1799,61 @@ impl Compiler {
         Ok(())
     }
 
+    fn optimize_instruction(&mut self) {
+        let instructions = self.current_instructions();
+        match instructions.pop().unwrap() {
+            Instruction::BinaryOperation { op, inplace } => {
+                macro_rules! lc {
+                    ($name:ident {$($field:tt)*}) => {
+                        Instruction::LoadConst {
+                            value: bytecode::Constant::$name {$($field)*},
+                        }
+                    };
+                    ($name:ident, $($value:tt)*) => {
+                        lc!($name { value: $($value)* })
+                    };
+                }
+                macro_rules! emitconst {
+                    ($($arg:tt)*) => {
+                        self.emit(lc!($($arg)*))
+                    };
+                }
+                macro_rules! op {
+                    ($op:ident) => {
+                        bytecode::BinaryOperator::$op
+                    };
+                }
+                let rhs = instructions.pop().unwrap();
+                let lhs = instructions.pop().unwrap();
+                match (op, lhs, rhs) {
+                    (op!(Add), lc!(Integer, lhs), lc!(Integer, rhs)) => {
+                        emitconst!(Integer, lhs + rhs)
+                    }
+                    (op!(Subtract), lc!(Integer, lhs), lc!(Integer, rhs)) => {
+                        emitconst!(Integer, lhs - rhs)
+                    }
+                    (op!(Add), lc!(Float, lhs), lc!(Float, rhs)) => emitconst!(Float, lhs + rhs),
+                    (op!(Subtract), lc!(Float, lhs), lc!(Float, rhs)) => {
+                        emitconst!(Float, lhs - rhs)
+                    }
+                    (op!(Power), lc!(Float, lhs), lc!(Float, rhs)) => {
+                        emitconst!(Float, lhs.powf(rhs))
+                    }
+                    (op!(Add), lc!(String, mut lhs), lc!(String, rhs)) => {
+                        lhs.push_str(&rhs);
+                        emitconst!(String, lhs);
+                    }
+                    (op, lhs, rhs) => {
+                        self.emit(lhs);
+                        self.emit(rhs);
+                        self.emit(Instruction::BinaryOperation { op, inplace });
+                    }
+                }
+            }
+            other => self.emit(other),
+        }
+    }
+
     // Scope helpers:
     fn enter_scope(&mut self) {
         // println!("Enter scope {:?}", self.scope_stack);
@@ -1831,6 +1887,10 @@ impl Compiler {
         self.code_object_stack.last_mut().unwrap()
     }
 
+    fn current_instructions(&mut self) -> &mut Vec<Instruction> {
+        &mut self.current_code_object().instructions
+    }
+
     // Generate a new label
     fn new_label(&mut self) -> Label {
         let l = self.nxt_label;

From e9ad1f64ee9d1837921849beda60f4d5d1c6ddd6 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 3 Aug 2019 22:02:29 -0500
Subject: [PATCH 032/459] Make peephole optimizer a stream processor

---
 Cargo.toml           |   1 +
 src/compile.rs       | 117 +++++++++++------------------------------
 src/lib.rs           |   2 +
 src/output_stream.rs |  40 ++++++++++++++
 src/peephole.rs      | 121 +++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 195 insertions(+), 86 deletions(-)
 create mode 100644 src/output_stream.rs
 create mode 100644 src/peephole.rs

diff --git a/Cargo.toml b/Cargo.toml
index 6531fc91..ef550980 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,3 +13,4 @@ rustpython-bytecode = { path = "../bytecode", version = "0.1.0" }
 rustpython-parser = { path = "../parser", version = "0.1.0" }
 num-complex = { version = "0.2", features = ["serde"] }
 log = "0.3"
+arrayvec = "0.4"
diff --git a/src/compile.rs b/src/compile.rs
index 2b7d34e7..37d66f03 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -6,13 +6,17 @@
 //!   https://github.com/micropython/micropython/blob/master/py/compile.c
 
 use crate::error::{CompileError, CompileErrorType};
+use crate::output_stream::{CodeObjectStream, OutputStream};
+use crate::peephole::PeepholeOptimizer;
 use crate::symboltable::{make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope};
 use num_complex::Complex64;
 use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Varargs};
 use rustpython_parser::{ast, parser};
 
-struct Compiler {
-    code_object_stack: Vec<CodeObject>,
+type BasicOutputStream = PeepholeOptimizer<CodeObjectStream>;
+
+struct Compiler<O: OutputStream = BasicOutputStream> {
+    output_stack: Vec<O>,
     scope_stack: Vec<SymbolScope>,
     nxt_label: usize,
     source_path: Option<String>,
@@ -109,18 +113,21 @@ enum EvalContext {
     Expression,
 }
 
-type Label = usize;
+pub(crate) type Label = usize;
 
-impl Default for Compiler {
+impl<O> Default for Compiler<O>
+where
+    O: OutputStream + Default,
+{
     fn default() -> Self {
         Compiler::new(0)
     }
 }
 
-impl Compiler {
+impl<O: OutputStream> Compiler<O> {
     fn new(optimize: u8) -> Self {
         Compiler {
-            code_object_stack: Vec::new(),
+            output_stack: Vec::new(),
             scope_stack: Vec::new(),
             nxt_label: 0,
             source_path: None,
@@ -132,9 +139,13 @@ impl Compiler {
         }
     }
 
+    fn push_output(&mut self, code: CodeObject) {
+        self.output_stack.push(code.into());
+    }
+
     fn push_new_code_object(&mut self, obj_name: String) {
         let line_number = self.get_source_line_number();
-        self.code_object_stack.push(CodeObject::new(
+        self.push_output(CodeObject::new(
             Vec::new(),
             Varargs::None,
             Vec::new(),
@@ -146,8 +157,7 @@ impl Compiler {
     }
 
     fn pop_code_object(&mut self) -> CodeObject {
-        // self.scope_stack.pop().unwrap();
-        self.code_object_stack.pop().unwrap()
+        self.output_stack.pop().unwrap().into()
     }
 
     fn compile_program(
@@ -155,10 +165,10 @@ impl Compiler {
         program: &ast::Program,
         symbol_scope: SymbolScope,
     ) -> Result<(), CompileError> {
-        let size_before = self.code_object_stack.len();
+        let size_before = self.output_stack.len();
         self.scope_stack.push(symbol_scope);
         self.compile_statements(&program.statements)?;
-        assert!(self.code_object_stack.len() == size_before);
+        assert_eq!(self.output_stack.len(), size_before);
 
         // Emit None at end:
         self.emit(Instruction::LoadConst {
@@ -636,7 +646,7 @@ impl Compiler {
         }
 
         let line_number = self.get_source_line_number();
-        self.code_object_stack.push(CodeObject::new(
+        self.push_output(CodeObject::new(
             args.args.iter().map(|a| a.arg.clone()).collect(),
             compile_varargs(&args.vararg),
             args.kwonlyargs.iter().map(|a| a.arg.clone()).collect(),
@@ -889,7 +899,7 @@ impl Compiler {
         self.prepare_decorators(decorator_list)?;
         self.emit(Instruction::LoadBuildClass);
         let line_number = self.get_source_line_number();
-        self.code_object_stack.push(CodeObject::new(
+        self.push_output(CodeObject::new(
             vec![],
             Varargs::None,
             vec![],
@@ -1481,7 +1491,6 @@ impl Compiler {
                 self.set_label(end_label);
             }
         }
-        self.optimize_instruction();
         Ok(())
     }
 
@@ -1620,7 +1629,7 @@ impl Compiler {
 
         let line_number = self.get_source_line_number();
         // Create magnificent function <listcomp>:
-        self.code_object_stack.push(CodeObject::new(
+        self.push_output(CodeObject::new(
             vec![".0".to_string()],
             Varargs::None,
             vec![],
@@ -1799,61 +1808,6 @@ impl Compiler {
         Ok(())
     }
 
-    fn optimize_instruction(&mut self) {
-        let instructions = self.current_instructions();
-        match instructions.pop().unwrap() {
-            Instruction::BinaryOperation { op, inplace } => {
-                macro_rules! lc {
-                    ($name:ident {$($field:tt)*}) => {
-                        Instruction::LoadConst {
-                            value: bytecode::Constant::$name {$($field)*},
-                        }
-                    };
-                    ($name:ident, $($value:tt)*) => {
-                        lc!($name { value: $($value)* })
-                    };
-                }
-                macro_rules! emitconst {
-                    ($($arg:tt)*) => {
-                        self.emit(lc!($($arg)*))
-                    };
-                }
-                macro_rules! op {
-                    ($op:ident) => {
-                        bytecode::BinaryOperator::$op
-                    };
-                }
-                let rhs = instructions.pop().unwrap();
-                let lhs = instructions.pop().unwrap();
-                match (op, lhs, rhs) {
-                    (op!(Add), lc!(Integer, lhs), lc!(Integer, rhs)) => {
-                        emitconst!(Integer, lhs + rhs)
-                    }
-                    (op!(Subtract), lc!(Integer, lhs), lc!(Integer, rhs)) => {
-                        emitconst!(Integer, lhs - rhs)
-                    }
-                    (op!(Add), lc!(Float, lhs), lc!(Float, rhs)) => emitconst!(Float, lhs + rhs),
-                    (op!(Subtract), lc!(Float, lhs), lc!(Float, rhs)) => {
-                        emitconst!(Float, lhs - rhs)
-                    }
-                    (op!(Power), lc!(Float, lhs), lc!(Float, rhs)) => {
-                        emitconst!(Float, lhs.powf(rhs))
-                    }
-                    (op!(Add), lc!(String, mut lhs), lc!(String, rhs)) => {
-                        lhs.push_str(&rhs);
-                        emitconst!(String, lhs);
-                    }
-                    (op, lhs, rhs) => {
-                        self.emit(lhs);
-                        self.emit(rhs);
-                        self.emit(Instruction::BinaryOperation { op, inplace });
-                    }
-                }
-            }
-            other => self.emit(other),
-        }
-    }
-
     // Scope helpers:
     fn enter_scope(&mut self) {
         // println!("Enter scope {:?}", self.scope_stack);
@@ -1877,18 +1831,11 @@ impl Compiler {
     // Low level helper functions:
     fn emit(&mut self, instruction: Instruction) {
         let location = compile_location(&self.current_source_location);
-        let cur_code_obj = self.current_code_object();
-        cur_code_obj.instructions.push(instruction);
-        cur_code_obj.locations.push(location);
         // TODO: insert source filename
-    }
-
-    fn current_code_object(&mut self) -> &mut CodeObject {
-        self.code_object_stack.last_mut().unwrap()
-    }
-
-    fn current_instructions(&mut self) -> &mut Vec<Instruction> {
-        &mut self.current_code_object().instructions
+        self.output_stack
+            .last_mut()
+            .unwrap()
+            .emit(instruction, location);
     }
 
     // Generate a new label
@@ -1900,9 +1847,7 @@ impl Compiler {
 
     // Assign current position the given label
     fn set_label(&mut self, label: Label) {
-        let position = self.current_code_object().instructions.len();
-        // assert!(label not in self.label_map)
-        self.current_code_object().label_map.insert(label, position);
+        self.output_stack.last_mut().unwrap().set_label(label)
     }
 
     fn set_source_location(&mut self, location: &ast::Location) {
@@ -1922,7 +1867,7 @@ impl Compiler {
     }
 
     fn mark_generator(&mut self) {
-        self.current_code_object().is_generator = true;
+        self.output_stack.last_mut().unwrap().mark_generator();
     }
 }
 
@@ -1971,7 +1916,7 @@ mod tests {
     use rustpython_parser::parser;
 
     fn compile_exec(source: &str) -> CodeObject {
-        let mut compiler: Compiler = Default::default();
+        let mut compiler = Compiler::default();
         compiler.source_path = Some("source_path".to_string());
         compiler.push_new_code_object("<module>".to_string());
         let ast = parser::parse_program(&source.to_string()).unwrap();
diff --git a/src/lib.rs b/src/lib.rs
index de949be2..0201e249 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -9,3 +9,5 @@ extern crate log;
 pub mod compile;
 pub mod error;
 pub mod symboltable;
+pub mod output_stream;
+pub mod peephole;
diff --git a/src/output_stream.rs b/src/output_stream.rs
new file mode 100644
index 00000000..35321dc5
--- /dev/null
+++ b/src/output_stream.rs
@@ -0,0 +1,40 @@
+use crate::compile::Label;
+use rustpython_bytecode::bytecode::{CodeObject, Instruction, Location};
+
+pub trait OutputStream: From<CodeObject> + Into<CodeObject> {
+    /// Output an instruction
+    fn emit(&mut self, instruction: Instruction, location: Location);
+    /// Set a label on an instruction
+    fn set_label(&mut self, label: Label);
+    /// Mark the inner CodeObject as a generator
+    fn mark_generator(&mut self);
+}
+
+pub struct CodeObjectStream {
+    code: CodeObject,
+}
+
+impl From<CodeObject> for CodeObjectStream {
+    fn from(code: CodeObject) -> Self {
+        CodeObjectStream { code }
+    }
+}
+impl From<CodeObjectStream> for CodeObject {
+    fn from(stream: CodeObjectStream) -> Self {
+        stream.code
+    }
+}
+
+impl OutputStream for CodeObjectStream {
+    fn emit(&mut self, instruction: Instruction, location: Location) {
+        self.code.instructions.push(instruction);
+        self.code.locations.push(location);
+    }
+    fn set_label(&mut self, label: Label) {
+        let position = self.code.instructions.len();
+        self.code.label_map.insert(label, position);
+    }
+    fn mark_generator(&mut self) {
+        self.code.is_generator = true;
+    }
+}
diff --git a/src/peephole.rs b/src/peephole.rs
new file mode 100644
index 00000000..0db95078
--- /dev/null
+++ b/src/peephole.rs
@@ -0,0 +1,121 @@
+use crate::output_stream::OutputStream;
+use arrayvec::ArrayVec;
+use rustpython_bytecode::bytecode::{self, CodeObject, Instruction, Location};
+
+const PEEPHOLE_BUFFER_SIZE: usize = 10;
+
+pub struct PeepholeOptimizer<O: OutputStream> {
+    inner: O,
+    buffer: ArrayVec<[(Instruction, Location); PEEPHOLE_BUFFER_SIZE]>,
+}
+
+impl<O: OutputStream> From<CodeObject> for PeepholeOptimizer<O> {
+    fn from(code: CodeObject) -> Self {
+        Self::new(code.into())
+    }
+}
+impl<O: OutputStream> From<PeepholeOptimizer<O>> for CodeObject {
+    fn from(mut peep: PeepholeOptimizer<O>) -> Self {
+        peep.flush();
+        peep.inner.into()
+    }
+}
+
+impl<O: OutputStream> PeepholeOptimizer<O> {
+    pub fn new(inner: O) -> Self {
+        PeepholeOptimizer {
+            inner,
+            buffer: ArrayVec::default(),
+        }
+    }
+
+    fn emit(&mut self, instruction: Instruction, loc: Location) {
+        if self.buffer.is_full() {
+            let (instr, loc) = self.buffer.remove(0);
+            self.inner.emit(instr, loc);
+            assert_eq!(self.buffer.len(), PEEPHOLE_BUFFER_SIZE - 1)
+        }
+        // safe because we just checked that: if full then remove one element from it
+        unsafe { self.buffer.push_unchecked((instruction, loc)) };
+    }
+
+    fn pop(&mut self) -> (Instruction, Location) {
+        self.buffer.pop().unwrap()
+    }
+
+    fn optimize(&mut self, instruction: Instruction, loc: Location) {
+        match instruction {
+            Instruction::BinaryOperation { op, inplace } => {
+                let (rhs, rhs_loc) = self.pop();
+                let (lhs, lhs_loc) = self.pop();
+                macro_rules! lc {
+                    ($name:ident {$($field:tt)*}) => {
+                        Instruction::LoadConst {
+                            value: bytecode::Constant::$name {$($field)*},
+                        }
+                    };
+                    ($name:ident, $($value:tt)*) => {
+                        lc!($name { value: $($value)* })
+                    };
+                }
+                macro_rules! emitconst {
+                    ($($arg:tt)*) => {
+                        self.emit(lc!($($arg)*), lhs_loc)
+                    };
+                }
+                macro_rules! op {
+                    ($op:ident) => {
+                        bytecode::BinaryOperator::$op
+                    };
+                }
+                match (op, lhs, rhs) {
+                    (op!(Add), lc!(Integer, lhs), lc!(Integer, rhs)) => {
+                        emitconst!(Integer, lhs + rhs)
+                    }
+                    (op!(Subtract), lc!(Integer, lhs), lc!(Integer, rhs)) => {
+                        emitconst!(Integer, lhs - rhs)
+                    }
+                    (op!(Add), lc!(Float, lhs), lc!(Float, rhs)) => emitconst!(Float, lhs + rhs),
+                    (op!(Subtract), lc!(Float, lhs), lc!(Float, rhs)) => {
+                        emitconst!(Float, lhs - rhs)
+                    }
+                    (op!(Power), lc!(Float, lhs), lc!(Float, rhs)) => {
+                        emitconst!(Float, lhs.powf(rhs))
+                    }
+                    (op!(Add), lc!(String, mut lhs), lc!(String, rhs)) => {
+                        lhs.push_str(&rhs);
+                        emitconst!(String, lhs);
+                    }
+                    (op, lhs, rhs) => {
+                        self.emit(lhs, lhs_loc);
+                        self.emit(rhs, rhs_loc);
+                        self.emit(Instruction::BinaryOperation { op, inplace }, loc);
+                    }
+                }
+            }
+            other => self.emit(other, loc),
+        }
+    }
+
+    fn flush(&mut self) {
+        for (instruction, location) in self.buffer.drain(..) {
+            self.inner.emit(instruction, location);
+        }
+    }
+}
+
+impl<O> OutputStream for PeepholeOptimizer<O>
+where
+    O: OutputStream,
+{
+    fn emit(&mut self, instruction: Instruction, location: Location) {
+        self.optimize(instruction, location);
+    }
+    fn set_label(&mut self, label: crate::compile::Label) {
+        self.flush();
+        self.inner.set_label(label);
+    }
+    fn mark_generator(&mut self) {
+        self.inner.mark_generator()
+    }
+}

From a01853fad5cb9e09a5e8b1ae8b9ffe6b24aa8ca0 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 3 Aug 2019 22:44:36 -0500
Subject: [PATCH 033/459] Add constant optimization test

---
 src/compile.rs  | 24 ++++++++++++++++++++++--
 src/peephole.rs |  4 ++++
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 37d66f03..ef481df0 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -117,7 +117,7 @@ pub(crate) type Label = usize;
 
 impl<O> Default for Compiler<O>
 where
-    O: OutputStream + Default,
+    O: OutputStream,
 {
     fn default() -> Self {
         Compiler::new(0)
@@ -1916,7 +1916,7 @@ mod tests {
     use rustpython_parser::parser;
 
     fn compile_exec(source: &str) -> CodeObject {
-        let mut compiler = Compiler::default();
+        let mut compiler: Compiler = Default::default();
         compiler.source_path = Some("source_path".to_string());
         compiler.push_new_code_object("<module>".to_string());
         let ast = parser::parse_program(&source.to_string()).unwrap();
@@ -2003,4 +2003,24 @@ mod tests {
             code.instructions
         );
     }
+
+    #[test]
+    fn test_constant_optimization() {
+        let code = compile_exec("1 + 2 + 3 + 4\n1.5 * 2.5");
+        assert_eq!(
+            code.instructions,
+            vec![
+                LoadConst {
+                    value: Integer { value: 10.into() }
+                },
+                Pop,
+                LoadConst {
+                    value: Float { value: 3.75 }
+                },
+                Pop,
+                LoadConst { value: None },
+                ReturnValue,
+            ]
+        );
+    }
 }
diff --git a/src/peephole.rs b/src/peephole.rs
index 0db95078..31aad96d 100644
--- a/src/peephole.rs
+++ b/src/peephole.rs
@@ -79,6 +79,10 @@ impl<O: OutputStream> PeepholeOptimizer<O> {
                     (op!(Subtract), lc!(Float, lhs), lc!(Float, rhs)) => {
                         emitconst!(Float, lhs - rhs)
                     }
+                    (op!(Multiply), lc!(Float, lhs), lc!(Float, rhs)) => {
+                        emitconst!(Float, lhs * rhs)
+                    }
+                    (op!(Divide), lc!(Float, lhs), lc!(Float, rhs)) => emitconst!(Float, lhs / rhs),
                     (op!(Power), lc!(Float, lhs), lc!(Float, rhs)) => {
                         emitconst!(Float, lhs.powf(rhs))
                     }

From 347d6c006aace8d408228fd2e1b62e82dc6bcfca Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 3 Aug 2019 23:30:14 -0500
Subject: [PATCH 034/459] Fix label issue

---
 src/lib.rs      |   2 +-
 src/peephole.rs | 101 +++++++++++++++++++++++++++++++++---------------
 2 files changed, 71 insertions(+), 32 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index 0201e249..c2a0e54d 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -8,6 +8,6 @@ extern crate log;
 
 pub mod compile;
 pub mod error;
-pub mod symboltable;
 pub mod output_stream;
 pub mod peephole;
+pub mod symboltable;
diff --git a/src/peephole.rs b/src/peephole.rs
index 31aad96d..adfd71e1 100644
--- a/src/peephole.rs
+++ b/src/peephole.rs
@@ -1,12 +1,31 @@
+use crate::compile::Label;
 use crate::output_stream::OutputStream;
 use arrayvec::ArrayVec;
 use rustpython_bytecode::bytecode::{self, CodeObject, Instruction, Location};
 
-const PEEPHOLE_BUFFER_SIZE: usize = 10;
+const PEEPHOLE_BUFFER_SIZE: usize = 20;
+
+struct InstructionMetadata {
+    loc: Location,
+    labels: Vec<Label>,
+}
+
+impl InstructionMetadata {
+    fn from_multiple(metas: Vec<Self>) -> Self {
+        debug_assert!(!metas.is_empty(), "`metas` must not be empty");
+        InstructionMetadata {
+            loc: metas[0].loc.clone(),
+            labels: metas
+                .into_iter()
+                .flat_map(|meta| meta.labels.into_iter())
+                .collect(),
+        }
+    }
+}
 
 pub struct PeepholeOptimizer<O: OutputStream> {
     inner: O,
-    buffer: ArrayVec<[(Instruction, Location); PEEPHOLE_BUFFER_SIZE]>,
+    buffer: ArrayVec<[(Instruction, InstructionMetadata); PEEPHOLE_BUFFER_SIZE]>,
 }
 
 impl<O: OutputStream> From<CodeObject> for PeepholeOptimizer<O> {
@@ -29,25 +48,31 @@ impl<O: OutputStream> PeepholeOptimizer<O> {
         }
     }
 
-    fn emit(&mut self, instruction: Instruction, loc: Location) {
+    fn inner_emit(inner: &mut O, instruction: Instruction, meta: InstructionMetadata) {
+        inner.emit(instruction, meta.loc);
+        for label in meta.labels {
+            inner.set_label(label);
+        }
+    }
+
+    fn emit(&mut self, instruction: Instruction, meta: InstructionMetadata) {
         if self.buffer.is_full() {
-            let (instr, loc) = self.buffer.remove(0);
-            self.inner.emit(instr, loc);
-            assert_eq!(self.buffer.len(), PEEPHOLE_BUFFER_SIZE - 1)
+            let (instr, meta) = self.buffer.remove(0);
+            Self::inner_emit(&mut self.inner, instr, meta);
         }
         // safe because we just checked that: if full then remove one element from it
-        unsafe { self.buffer.push_unchecked((instruction, loc)) };
+        unsafe { self.buffer.push_unchecked((instruction, meta)) };
     }
 
-    fn pop(&mut self) -> (Instruction, Location) {
+    fn pop(&mut self) -> (Instruction, InstructionMetadata) {
         self.buffer.pop().unwrap()
     }
 
-    fn optimize(&mut self, instruction: Instruction, loc: Location) {
+    fn optimize(&mut self, instruction: Instruction, meta: InstructionMetadata) {
         match instruction {
             Instruction::BinaryOperation { op, inplace } => {
-                let (rhs, rhs_loc) = self.pop();
-                let (lhs, lhs_loc) = self.pop();
+                let (rhs, rhs_meta) = self.pop();
+                let (lhs, lhs_meta) = self.pop();
                 macro_rules! lc {
                     ($name:ident {$($field:tt)*}) => {
                         Instruction::LoadConst {
@@ -59,8 +84,11 @@ impl<O: OutputStream> PeepholeOptimizer<O> {
                     };
                 }
                 macro_rules! emitconst {
-                    ($($arg:tt)*) => {
-                        self.emit(lc!($($arg)*), lhs_loc)
+                    ([$($metas:expr),*], $($arg:tt)*) => {
+                        self.emit(
+                            lc!($($arg)*),
+                            InstructionMetadata::from_multiple(vec![$($metas),*]),
+                        )
                     };
                 }
                 macro_rules! op {
@@ -70,40 +98,44 @@ impl<O: OutputStream> PeepholeOptimizer<O> {
                 }
                 match (op, lhs, rhs) {
                     (op!(Add), lc!(Integer, lhs), lc!(Integer, rhs)) => {
-                        emitconst!(Integer, lhs + rhs)
+                        emitconst!([lhs_meta, rhs_meta], Integer, lhs + rhs)
                     }
                     (op!(Subtract), lc!(Integer, lhs), lc!(Integer, rhs)) => {
-                        emitconst!(Integer, lhs - rhs)
+                        emitconst!([lhs_meta, rhs_meta], Integer, lhs - rhs)
+                    }
+                    (op!(Add), lc!(Float, lhs), lc!(Float, rhs)) => {
+                        emitconst!([lhs_meta, rhs_meta], Float, lhs + rhs)
                     }
-                    (op!(Add), lc!(Float, lhs), lc!(Float, rhs)) => emitconst!(Float, lhs + rhs),
                     (op!(Subtract), lc!(Float, lhs), lc!(Float, rhs)) => {
-                        emitconst!(Float, lhs - rhs)
+                        emitconst!([lhs_meta, rhs_meta], Float, lhs - rhs)
                     }
                     (op!(Multiply), lc!(Float, lhs), lc!(Float, rhs)) => {
-                        emitconst!(Float, lhs * rhs)
+                        emitconst!([lhs_meta, rhs_meta], Float, lhs * rhs)
+                    }
+                    (op!(Divide), lc!(Float, lhs), lc!(Float, rhs)) => {
+                        emitconst!([lhs_meta, rhs_meta], Float, lhs / rhs)
                     }
-                    (op!(Divide), lc!(Float, lhs), lc!(Float, rhs)) => emitconst!(Float, lhs / rhs),
                     (op!(Power), lc!(Float, lhs), lc!(Float, rhs)) => {
-                        emitconst!(Float, lhs.powf(rhs))
+                        emitconst!([lhs_meta, rhs_meta], Float, lhs.powf(rhs))
                     }
                     (op!(Add), lc!(String, mut lhs), lc!(String, rhs)) => {
                         lhs.push_str(&rhs);
-                        emitconst!(String, lhs);
+                        emitconst!([lhs_meta, rhs_meta], String, lhs);
                     }
                     (op, lhs, rhs) => {
-                        self.emit(lhs, lhs_loc);
-                        self.emit(rhs, rhs_loc);
-                        self.emit(Instruction::BinaryOperation { op, inplace }, loc);
+                        self.emit(lhs, lhs_meta);
+                        self.emit(rhs, rhs_meta);
+                        self.emit(Instruction::BinaryOperation { op, inplace }, meta);
                     }
                 }
             }
-            other => self.emit(other, loc),
+            other => self.emit(other, meta),
         }
     }
 
     fn flush(&mut self) {
-        for (instruction, location) in self.buffer.drain(..) {
-            self.inner.emit(instruction, location);
+        for (instruction, meta) in self.buffer.drain(..) {
+            Self::inner_emit(&mut self.inner, instruction, meta);
         }
     }
 }
@@ -112,12 +144,19 @@ impl<O> OutputStream for PeepholeOptimizer<O>
 where
     O: OutputStream,
 {
-    fn emit(&mut self, instruction: Instruction, location: Location) {
-        self.optimize(instruction, location);
+    fn emit(&mut self, instruction: Instruction, loc: Location) {
+        self.optimize(
+            instruction,
+            InstructionMetadata {
+                loc,
+                labels: Vec::new(),
+            },
+        );
     }
     fn set_label(&mut self, label: crate::compile::Label) {
-        self.flush();
-        self.inner.set_label(label);
+        if let Some(instr) = self.buffer.last_mut() {
+            instr.1.labels.push(label)
+        }
     }
     fn mark_generator(&mut self) {
         self.inner.mark_generator()

From 4cb8cc34e48d8ffe5ae7d6ed1204cdbd4adceb11 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sun, 4 Aug 2019 01:22:51 -0500
Subject: [PATCH 035/459] Add tuple unpacking optimization

---
 src/peephole.rs | 58 ++++++++++++++++++++++++++++++++-----------------
 1 file changed, 38 insertions(+), 20 deletions(-)

diff --git a/src/peephole.rs b/src/peephole.rs
index adfd71e1..c0bc5a0b 100644
--- a/src/peephole.rs
+++ b/src/peephole.rs
@@ -10,8 +10,8 @@ struct InstructionMetadata {
     labels: Vec<Label>,
 }
 
-impl InstructionMetadata {
-    fn from_multiple(metas: Vec<Self>) -> Self {
+impl From<Vec<InstructionMetadata>> for InstructionMetadata {
+    fn from(metas: Vec<Self>) -> Self {
         debug_assert!(!metas.is_empty(), "`metas` must not be empty");
         InstructionMetadata {
             loc: metas[0].loc.clone(),
@@ -69,28 +69,28 @@ impl<O: OutputStream> PeepholeOptimizer<O> {
     }
 
     fn optimize(&mut self, instruction: Instruction, meta: InstructionMetadata) {
+        macro_rules! lc {
+            ($name:ident {$($field:tt)*}) => {
+                Instruction::LoadConst {
+                    value: bytecode::Constant::$name {$($field)*},
+                }
+            };
+            ($name:ident, $($value:tt)*) => {
+                lc!($name { value: $($value)* })
+            };
+        }
+        macro_rules! emitconst {
+            ([$($metas:expr),*], $($arg:tt)*) => {
+                self.emit(
+                    lc!($($arg)*),
+                    InstructionMetadata::from(vec![$($metas),*]),
+                )
+            };
+        }
         match instruction {
             Instruction::BinaryOperation { op, inplace } => {
                 let (rhs, rhs_meta) = self.pop();
                 let (lhs, lhs_meta) = self.pop();
-                macro_rules! lc {
-                    ($name:ident {$($field:tt)*}) => {
-                        Instruction::LoadConst {
-                            value: bytecode::Constant::$name {$($field)*},
-                        }
-                    };
-                    ($name:ident, $($value:tt)*) => {
-                        lc!($name { value: $($value)* })
-                    };
-                }
-                macro_rules! emitconst {
-                    ([$($metas:expr),*], $($arg:tt)*) => {
-                        self.emit(
-                            lc!($($arg)*),
-                            InstructionMetadata::from_multiple(vec![$($metas),*]),
-                        )
-                    };
-                }
                 macro_rules! op {
                     ($op:ident) => {
                         bytecode::BinaryOperator::$op
@@ -129,6 +129,24 @@ impl<O: OutputStream> PeepholeOptimizer<O> {
                     }
                 }
             }
+            Instruction::UnpackSequence { size } => {
+                let (arg, arg_meta) = self.pop();
+                match arg {
+                    Instruction::BuildTuple {
+                        size: tup_size,
+                        unpack,
+                    } if !unpack && tup_size == size => {
+                        self.emit(
+                            Instruction::Reverse { amount: size },
+                            vec![arg_meta, meta].into(),
+                        );
+                    }
+                    arg => {
+                        self.emit(arg, arg_meta);
+                        self.emit(instruction, meta);
+                    }
+                }
+            }
             other => self.emit(other, meta),
         }
     }

From b22698a6f14dc5e4e32745e3b960008f8cd5dfaf Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Wed, 7 Aug 2019 15:31:24 +0200
Subject: [PATCH 036/459] Add variable annotation syntax. Simplify grammar for
 comprehensions. Add capital J suffix for complex numbers.

---
 src/compile.rs     | 14 ++++++++++++++
 src/symboltable.rs | 11 +++++++++++
 2 files changed, 25 insertions(+)

diff --git a/src/compile.rs b/src/compile.rs
index 3fa6bdbe..8d5c4c47 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -553,6 +553,20 @@ impl Compiler {
                 self.compile_op(op, true);
                 self.compile_store(target)?;
             }
+            AnnAssign {
+                target,
+                annotation,
+                value,
+            } => {
+                // Throw away annotation:
+                self.compile_expression(annotation)?;
+                self.emit(Instruction::Pop);
+
+                if let Some(value) = value {
+                    self.compile_expression(value)?;
+                    self.compile_store(target)?;
+                }
+            }
             Delete { targets } => {
                 for target in targets {
                     self.compile_delete(target)?;
diff --git a/src/symboltable.rs b/src/symboltable.rs
index b12e0f92..c1db24bf 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -335,6 +335,17 @@ impl SymbolTableBuilder {
                 self.scan_expression(target)?;
                 self.scan_expression(value)?;
             }
+            AnnAssign {
+                target,
+                annotation,
+                value,
+            } => {
+                self.scan_expression(target)?;
+                self.scan_expression(annotation)?;
+                if let Some(value) = value {
+                    self.scan_expression(value)?;
+                }
+            }
             With { items, body, .. } => {
                 for item in items {
                     self.scan_expression(&item.context_expr)?;

From b6dd92daf61eac069ebfe68ae5c7cb8d9e4b727e Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Wed, 7 Aug 2019 16:13:12 +0200
Subject: [PATCH 037/459] Properly fill the __annotations__ dictionary.

---
 src/compile.rs | 108 ++++++++++++++++++++++++++++++++-----------------
 1 file changed, 71 insertions(+), 37 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 8d5c4c47..f7ac924b 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -358,33 +358,7 @@ impl Compiler {
                 }
                 self.set_label(end_label);
             }
-            While { test, body, orelse } => {
-                let start_label = self.new_label();
-                let else_label = self.new_label();
-                let end_label = self.new_label();
-                self.emit(Instruction::SetupLoop {
-                    start: start_label,
-                    end: end_label,
-                });
-
-                self.set_label(start_label);
-
-                self.compile_test(test, None, Some(else_label), EvalContext::Statement)?;
-
-                let was_in_loop = self.in_loop;
-                self.in_loop = true;
-                self.compile_statements(body)?;
-                self.in_loop = was_in_loop;
-                self.emit(Instruction::Jump {
-                    target: start_label,
-                });
-                self.set_label(else_label);
-                self.emit(Instruction::PopBlock);
-                if let Some(orelse) = orelse {
-                    self.compile_statements(orelse)?;
-                }
-                self.set_label(end_label);
-            }
+            While { test, body, orelse } => self.compile_while(test, body, orelse)?,
             With {
                 is_async,
                 items,
@@ -557,16 +531,7 @@ impl Compiler {
                 target,
                 annotation,
                 value,
-            } => {
-                // Throw away annotation:
-                self.compile_expression(annotation)?;
-                self.emit(Instruction::Pop);
-
-                if let Some(value) = value {
-                    self.compile_expression(value)?;
-                    self.compile_store(target)?;
-                }
-            }
+            } => self.compile_annotated_assign(target, annotation, value)?,
             Delete { targets } => {
                 for target in targets {
                     self.compile_delete(target)?;
@@ -1015,6 +980,40 @@ impl Compiler {
         }
     }
 
+    fn compile_while(
+        &mut self,
+        test: &ast::Expression,
+        body: &[ast::Statement],
+        orelse: &Option<Vec<ast::Statement>>,
+    ) -> Result<(), CompileError> {
+        let start_label = self.new_label();
+        let else_label = self.new_label();
+        let end_label = self.new_label();
+        self.emit(Instruction::SetupLoop {
+            start: start_label,
+            end: end_label,
+        });
+
+        self.set_label(start_label);
+
+        self.compile_test(test, None, Some(else_label), EvalContext::Statement)?;
+
+        let was_in_loop = self.in_loop;
+        self.in_loop = true;
+        self.compile_statements(body)?;
+        self.in_loop = was_in_loop;
+        self.emit(Instruction::Jump {
+            target: start_label,
+        });
+        self.set_label(else_label);
+        self.emit(Instruction::PopBlock);
+        if let Some(orelse) = orelse {
+            self.compile_statements(orelse)?;
+        }
+        self.set_label(end_label);
+        Ok(())
+    }
+
     fn compile_for(
         &mut self,
         target: &ast::Expression,
@@ -1133,6 +1132,39 @@ impl Compiler {
         Ok(())
     }
 
+    fn compile_annotated_assign(
+        &mut self,
+        target: &ast::Expression,
+        annotation: &ast::Expression,
+        value: &Option<ast::Expression>,
+    ) -> Result<(), CompileError> {
+        if let Some(value) = value {
+            self.compile_expression(value)?;
+            self.compile_store(target)?;
+        }
+
+        // Compile annotation:
+        self.compile_expression(annotation)?;
+
+        if let ast::ExpressionType::Identifier { name } = &target.node {
+            // Store as dict entry in __annotations__ dict:
+            self.emit(Instruction::LoadName {
+                name: String::from("__annotations__"),
+                scope: bytecode::NameScope::Local,
+            });
+            self.emit(Instruction::LoadConst {
+                value: bytecode::Constant::String {
+                    value: name.to_string(),
+                },
+            });
+            self.emit(Instruction::StoreSubscript);
+        } else {
+            // Drop annotation if not assigned to simple identifier.
+            self.emit(Instruction::Pop);
+        }
+        Ok(())
+    }
+
     fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> {
         match &target.node {
             ast::ExpressionType::Identifier { name } => {
@@ -1276,6 +1308,8 @@ impl Compiler {
 
     fn compile_expression(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
         trace!("Compiling {:?}", expression);
+        self.set_source_location(&expression.location);
+
         use ast::ExpressionType::*;
         match &expression.node {
             Call {

From 69b709e2b147679d24d1dde0f642a7c6ff5ca57d Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Wed, 7 Aug 2019 20:11:27 +0200
Subject: [PATCH 038/459] Add parser support for async for loop. Add yield
 expression inside parentheses.

---
 src/compile.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/compile.rs b/src/compile.rs
index f7ac924b..c3870a3a 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1702,6 +1702,10 @@ impl Compiler {
 
         let mut loop_labels = vec![];
         for generator in generators {
+            if generator.is_async {
+                unimplemented!("async for comprehensions");
+            }
+
             if loop_labels.is_empty() {
                 // Load iterator onto stack (passed as first argument):
                 self.emit(Instruction::LoadName {

From 70c5c240e7170e3d8dae2454e32ab02deeb03f92 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 7 Aug 2019 14:40:16 -0500
Subject: [PATCH 039/459] Address comments

---
 src/compile.rs  |  11 ++-
 src/lib.rs      |   2 +-
 src/peephole.rs | 224 +++++++++++++++++++++++++++---------------------
 3 files changed, 136 insertions(+), 101 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index ef481df0..f89772ec 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1832,10 +1832,13 @@ impl<O: OutputStream> Compiler<O> {
     fn emit(&mut self, instruction: Instruction) {
         let location = compile_location(&self.current_source_location);
         // TODO: insert source filename
+        self.current_output().emit(instruction, location);
+    }
+
+    fn current_output(&mut self) -> &mut O {
         self.output_stack
             .last_mut()
-            .unwrap()
-            .emit(instruction, location);
+            .expect("No OutputStream on stack")
     }
 
     // Generate a new label
@@ -1847,7 +1850,7 @@ impl<O: OutputStream> Compiler<O> {
 
     // Assign current position the given label
     fn set_label(&mut self, label: Label) {
-        self.output_stack.last_mut().unwrap().set_label(label)
+        self.current_output().set_label(label)
     }
 
     fn set_source_location(&mut self, location: &ast::Location) {
@@ -1867,7 +1870,7 @@ impl<O: OutputStream> Compiler<O> {
     }
 
     fn mark_generator(&mut self) {
-        self.output_stack.last_mut().unwrap().mark_generator();
+        self.current_output().mark_generator();
     }
 }
 
diff --git a/src/lib.rs b/src/lib.rs
index c2a0e54d..9e18f1c2 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -8,6 +8,6 @@ extern crate log;
 
 pub mod compile;
 pub mod error;
-pub mod output_stream;
+pub(crate) mod output_stream;
 pub mod peephole;
 pub mod symboltable;
diff --git a/src/peephole.rs b/src/peephole.rs
index c0bc5a0b..64c06b81 100644
--- a/src/peephole.rs
+++ b/src/peephole.rs
@@ -5,7 +5,7 @@ use rustpython_bytecode::bytecode::{self, CodeObject, Instruction, Location};
 
 const PEEPHOLE_BUFFER_SIZE: usize = 20;
 
-struct InstructionMetadata {
+pub struct InstructionMetadata {
     loc: Location,
     labels: Vec<Label>,
 }
@@ -22,8 +22,16 @@ impl From<Vec<InstructionMetadata>> for InstructionMetadata {
         }
     }
 }
+impl From<Location> for InstructionMetadata {
+    fn from(loc: Location) -> Self {
+        InstructionMetadata {
+            loc,
+            labels: Vec::new(),
+        }
+    }
+}
 
-pub struct PeepholeOptimizer<O: OutputStream> {
+pub(crate) struct PeepholeOptimizer<O: OutputStream> {
     inner: O,
     buffer: ArrayVec<[(Instruction, InstructionMetadata); PEEPHOLE_BUFFER_SIZE]>,
 }
@@ -55,100 +63,18 @@ impl<O: OutputStream> PeepholeOptimizer<O> {
         }
     }
 
-    fn emit(&mut self, instruction: Instruction, meta: InstructionMetadata) {
+    fn push(&mut self, instruction: Instruction, meta: InstructionMetadata) {
         if self.buffer.is_full() {
             let (instr, meta) = self.buffer.remove(0);
             Self::inner_emit(&mut self.inner, instr, meta);
         }
-        // safe because we just checked that: if full then remove one element from it
-        unsafe { self.buffer.push_unchecked((instruction, meta)) };
+        self.buffer.push((instruction, meta));
     }
 
     fn pop(&mut self) -> (Instruction, InstructionMetadata) {
-        self.buffer.pop().unwrap()
-    }
-
-    fn optimize(&mut self, instruction: Instruction, meta: InstructionMetadata) {
-        macro_rules! lc {
-            ($name:ident {$($field:tt)*}) => {
-                Instruction::LoadConst {
-                    value: bytecode::Constant::$name {$($field)*},
-                }
-            };
-            ($name:ident, $($value:tt)*) => {
-                lc!($name { value: $($value)* })
-            };
-        }
-        macro_rules! emitconst {
-            ([$($metas:expr),*], $($arg:tt)*) => {
-                self.emit(
-                    lc!($($arg)*),
-                    InstructionMetadata::from(vec![$($metas),*]),
-                )
-            };
-        }
-        match instruction {
-            Instruction::BinaryOperation { op, inplace } => {
-                let (rhs, rhs_meta) = self.pop();
-                let (lhs, lhs_meta) = self.pop();
-                macro_rules! op {
-                    ($op:ident) => {
-                        bytecode::BinaryOperator::$op
-                    };
-                }
-                match (op, lhs, rhs) {
-                    (op!(Add), lc!(Integer, lhs), lc!(Integer, rhs)) => {
-                        emitconst!([lhs_meta, rhs_meta], Integer, lhs + rhs)
-                    }
-                    (op!(Subtract), lc!(Integer, lhs), lc!(Integer, rhs)) => {
-                        emitconst!([lhs_meta, rhs_meta], Integer, lhs - rhs)
-                    }
-                    (op!(Add), lc!(Float, lhs), lc!(Float, rhs)) => {
-                        emitconst!([lhs_meta, rhs_meta], Float, lhs + rhs)
-                    }
-                    (op!(Subtract), lc!(Float, lhs), lc!(Float, rhs)) => {
-                        emitconst!([lhs_meta, rhs_meta], Float, lhs - rhs)
-                    }
-                    (op!(Multiply), lc!(Float, lhs), lc!(Float, rhs)) => {
-                        emitconst!([lhs_meta, rhs_meta], Float, lhs * rhs)
-                    }
-                    (op!(Divide), lc!(Float, lhs), lc!(Float, rhs)) => {
-                        emitconst!([lhs_meta, rhs_meta], Float, lhs / rhs)
-                    }
-                    (op!(Power), lc!(Float, lhs), lc!(Float, rhs)) => {
-                        emitconst!([lhs_meta, rhs_meta], Float, lhs.powf(rhs))
-                    }
-                    (op!(Add), lc!(String, mut lhs), lc!(String, rhs)) => {
-                        lhs.push_str(&rhs);
-                        emitconst!([lhs_meta, rhs_meta], String, lhs);
-                    }
-                    (op, lhs, rhs) => {
-                        self.emit(lhs, lhs_meta);
-                        self.emit(rhs, rhs_meta);
-                        self.emit(Instruction::BinaryOperation { op, inplace }, meta);
-                    }
-                }
-            }
-            Instruction::UnpackSequence { size } => {
-                let (arg, arg_meta) = self.pop();
-                match arg {
-                    Instruction::BuildTuple {
-                        size: tup_size,
-                        unpack,
-                    } if !unpack && tup_size == size => {
-                        self.emit(
-                            Instruction::Reverse { amount: size },
-                            vec![arg_meta, meta].into(),
-                        );
-                    }
-                    arg => {
-                        self.emit(arg, arg_meta);
-                        self.emit(instruction, meta);
-                    }
-                }
-            }
-            other => self.emit(other, meta),
-        }
+        self.buffer
+            .pop()
+            .expect("Failed to pop instruction from PeepholeOptimizer buffer")
     }
 
     fn flush(&mut self) {
@@ -163,13 +89,8 @@ where
     O: OutputStream,
 {
     fn emit(&mut self, instruction: Instruction, loc: Location) {
-        self.optimize(
-            instruction,
-            InstructionMetadata {
-                loc,
-                labels: Vec::new(),
-            },
-        );
+        self.push(instruction, loc.into());
+        optimize(self);
     }
     fn set_label(&mut self, label: crate::compile::Label) {
         if let Some(instr) = self.buffer.last_mut() {
@@ -180,3 +101,114 @@ where
         self.inner.mark_generator()
     }
 }
+
+impl<O: OutputStream> OptimizationBuffer for PeepholeOptimizer<O> {
+    fn emit(&mut self, instruction: Instruction, meta: InstructionMetadata) {
+        self.push(instruction, meta);
+    }
+    fn pop(&mut self) -> (Instruction, InstructionMetadata) {
+        self.pop()
+    }
+}
+
+// OPTIMIZATION
+
+pub trait OptimizationBuffer {
+    fn emit(&mut self, instruction: Instruction, meta: InstructionMetadata);
+    fn pop(&mut self) -> (Instruction, InstructionMetadata);
+}
+
+macro_rules! lc {
+    ($name:ident {$($field:tt)*}) => {
+        Instruction::LoadConst {
+            value: bytecode::Constant::$name {$($field)*},
+        }
+    };
+    ($name:ident, $($value:tt)*) => {
+        lc!($name { value: $($value)* })
+    };
+}
+macro_rules! emitconst {
+    ($buf:expr, [$($metas:expr),*], $($arg:tt)*) => {
+        $buf.emit(
+            lc!($($arg)*),
+            InstructionMetadata::from(vec![$($metas),*]),
+        )
+    };
+}
+
+pub fn optimize(buf: &mut impl OptimizationBuffer) {
+    optimize_operator(buf);
+    optimize_unpack(buf);
+}
+
+fn optimize_operator(buf: &mut impl OptimizationBuffer) {
+    let (instruction, meta) = buf.pop();
+    if let Instruction::BinaryOperation { op, inplace } = instruction {
+        let (rhs, rhs_meta) = buf.pop();
+        let (lhs, lhs_meta) = buf.pop();
+        macro_rules! op {
+            ($op:ident) => {
+                bytecode::BinaryOperator::$op
+            };
+        }
+        match (op, lhs, rhs) {
+            (op!(Add), lc!(Integer, lhs), lc!(Integer, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Integer, lhs + rhs)
+            }
+            (op!(Subtract), lc!(Integer, lhs), lc!(Integer, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Integer, lhs - rhs)
+            }
+            (op!(Add), lc!(Float, lhs), lc!(Float, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs + rhs)
+            }
+            (op!(Subtract), lc!(Float, lhs), lc!(Float, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs - rhs)
+            }
+            (op!(Multiply), lc!(Float, lhs), lc!(Float, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs * rhs)
+            }
+            (op!(Divide), lc!(Float, lhs), lc!(Float, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs / rhs)
+            }
+            (op!(Power), lc!(Float, lhs), lc!(Float, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs.powf(rhs))
+            }
+            (op!(Add), lc!(String, mut lhs), lc!(String, rhs)) => {
+                lhs.push_str(&rhs);
+                emitconst!(buf, [lhs_meta, rhs_meta], String, lhs);
+            }
+            (op, lhs, rhs) => {
+                buf.emit(lhs, lhs_meta);
+                buf.emit(rhs, rhs_meta);
+                buf.emit(Instruction::BinaryOperation { op, inplace }, meta);
+            }
+        }
+    } else {
+        buf.emit(instruction, meta)
+    }
+}
+
+fn optimize_unpack(buf: &mut impl OptimizationBuffer) {
+    let (instruction, meta) = buf.pop();
+    if let Instruction::UnpackSequence { size } = instruction {
+        let (arg, arg_meta) = buf.pop();
+        match arg {
+            Instruction::BuildTuple {
+                size: tup_size,
+                unpack,
+            } if !unpack && tup_size == size => {
+                buf.emit(
+                    Instruction::Reverse { amount: size },
+                    vec![arg_meta, meta].into(),
+                );
+            }
+            arg => {
+                buf.emit(arg, arg_meta);
+                buf.emit(instruction, meta);
+            }
+        }
+    } else {
+        buf.emit(instruction, meta)
+    }
+}

From 5c1bc6756c3f42e12332771e859ee05a90797901 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 9 Aug 2019 21:36:42 -0500
Subject: [PATCH 040/459] Fix compilation of nested boolean operations

---
 src/compile.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index a13d3617..aafab376 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1303,11 +1303,9 @@ impl<O: OutputStream> Compiler<O> {
                             self.emit(Instruction::Pop);
                         }
                         if let Some(false_label) = false_label {
-                            self.emit(Instruction::Duplicate);
                             self.emit(Instruction::JumpIfFalse {
                                 target: false_label,
                             });
-                            self.emit(Instruction::Pop);
                         }
                     }
                 }

From fde382409c5d5d30c3a26c9dcb3ca934f0ecc3ab Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sun, 11 Aug 2019 12:29:10 -0500
Subject: [PATCH 041/459] Revert "Fix compilation of nested boolean operations"

---
 src/compile.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/compile.rs b/src/compile.rs
index aafab376..a13d3617 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1303,9 +1303,11 @@ impl<O: OutputStream> Compiler<O> {
                             self.emit(Instruction::Pop);
                         }
                         if let Some(false_label) = false_label {
+                            self.emit(Instruction::Duplicate);
                             self.emit(Instruction::JumpIfFalse {
                                 target: false_label,
                             });
+                            self.emit(Instruction::Pop);
                         }
                     }
                 }

From 4f9f9252e4b54ffa75474eb4603b5651e1901587 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Sun, 11 Aug 2019 20:20:15 +0200
Subject: [PATCH 042/459] Improve the situation regarding boolean operations.

---
 src/compile.rs     | 185 +++++++++++++++++++++++++++------------------
 src/symboltable.rs |   5 +-
 2 files changed, 112 insertions(+), 78 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index a13d3617..a0f20c1e 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -15,6 +15,7 @@ use rustpython_parser::{ast, parser};
 
 type BasicOutputStream = PeepholeOptimizer<CodeObjectStream>;
 
+/// Main structure holding the state of compilation.
 struct Compiler<O: OutputStream = BasicOutputStream> {
     output_stack: Vec<O>,
     scope_stack: Vec<SymbolScope>,
@@ -107,12 +108,6 @@ pub enum Mode {
     Single,
 }
 
-#[derive(Clone, Copy)]
-enum EvalContext {
-    Statement,
-    Expression,
-}
-
 pub(crate) type Label = usize;
 
 impl<O> Default for Compiler<O>
@@ -350,14 +345,14 @@ impl<O: OutputStream> Compiler<O> {
                 match orelse {
                     None => {
                         // Only if:
-                        self.compile_test(test, None, Some(end_label), EvalContext::Statement)?;
+                        self.compile_jump_if(test, false, end_label)?;
                         self.compile_statements(body)?;
                         self.set_label(end_label);
                     }
                     Some(statements) => {
                         // if - else:
                         let else_label = self.new_label();
-                        self.compile_test(test, None, Some(else_label), EvalContext::Statement)?;
+                        self.compile_jump_if(test, false, else_label)?;
                         self.compile_statements(body)?;
                         self.emit(Instruction::Jump { target: end_label });
 
@@ -459,7 +454,7 @@ impl<O: OutputStream> Compiler<O> {
                 // if some flag, ignore all assert statements!
                 if self.optimize == 0 {
                     let end_label = self.new_label();
-                    self.compile_test(test, Some(end_label), None, EvalContext::Statement)?;
+                    self.compile_jump_if(test, true, end_label)?;
                     self.emit(Instruction::LoadName {
                         name: String::from("AssertionError"),
                         scope: bytecode::NameScope::Local,
@@ -1006,7 +1001,7 @@ impl<O: OutputStream> Compiler<O> {
 
         self.set_label(start_label);
 
-        self.compile_test(test, None, Some(else_label), EvalContext::Statement)?;
+        self.compile_jump_if(test, false, else_label)?;
 
         let was_in_loop = self.in_loop;
         self.in_loop = true;
@@ -1118,12 +1113,9 @@ impl<O: OutputStream> Compiler<O> {
             });
 
             // if comparison result is false, we break with this value; if true, try the next one.
-            // (CPython compresses these three opcodes into JUMP_IF_FALSE_OR_POP)
-            self.emit(Instruction::Duplicate);
-            self.emit(Instruction::JumpIfFalse {
+            self.emit(Instruction::JumpIfFalseOrPop {
                 target: break_label,
             });
-            self.emit(Instruction::Pop);
         }
 
         // handle the last comparison
@@ -1256,66 +1248,120 @@ impl<O: OutputStream> Compiler<O> {
         self.emit(Instruction::BinaryOperation { op: i, inplace });
     }
 
-    fn compile_test(
+    /// Implement boolean short circuit evaluation logic.
+    /// https://en.wikipedia.org/wiki/Short-circuit_evaluation
+    ///
+    /// This means, in a boolean statement 'x and y' the variable y will
+    /// not be evaluated when x is false.
+    ///
+    /// The idea is to jump to a label if the expression is either true or false
+    /// (indicated by the condition parameter).
+    fn compile_jump_if(
         &mut self,
         expression: &ast::Expression,
-        true_label: Option<Label>,
-        false_label: Option<Label>,
-        context: EvalContext,
+        condition: bool,
+        target_label: Label,
     ) -> Result<(), CompileError> {
         // Compile expression for test, and jump to label if false
         match &expression.node {
-            ast::ExpressionType::BoolOp { a, op, b } => match op {
-                ast::BooleanOperator::And => {
-                    let f = false_label.unwrap_or_else(|| self.new_label());
-                    self.compile_test(a, None, Some(f), context)?;
-                    self.compile_test(b, true_label, false_label, context)?;
-                    if false_label.is_none() {
-                        self.set_label(f);
+            ast::ExpressionType::BoolOp { op, values } => {
+                match op {
+                    ast::BooleanOperator::And => {
+                        if condition {
+                            // If all values are true.
+                            let end_label = self.new_label();
+                            let (last_value, values) = values.split_last().unwrap();
+
+                            // If any of the values is false, we can short-circuit.
+                            for value in values {
+                                self.compile_jump_if(value, false, end_label)?;
+                            }
+
+                            // It depends upon the last value now: will it be true?
+                            self.compile_jump_if(last_value, true, target_label)?;
+                            self.set_label(end_label);
+                        } else {
+                            // If any value is false, the whole condition is false.
+                            for value in values {
+                                self.compile_jump_if(value, false, target_label)?;
+                            }
+                        }
                     }
-                }
-                ast::BooleanOperator::Or => {
-                    let t = true_label.unwrap_or_else(|| self.new_label());
-                    self.compile_test(a, Some(t), None, context)?;
-                    self.compile_test(b, true_label, false_label, context)?;
-                    if true_label.is_none() {
-                        self.set_label(t);
+                    ast::BooleanOperator::Or => {
+                        if condition {
+                            // If any of the values is true.
+                            for value in values {
+                                self.compile_jump_if(value, true, target_label)?;
+                            }
+                        } else {
+                            // If all of the values are false.
+                            let end_label = self.new_label();
+                            let (last_value, values) = values.split_last().unwrap();
+
+                            // If any value is true, we can short-circuit:
+                            for value in values {
+                                self.compile_jump_if(value, true, end_label)?;
+                            }
+
+                            // It all depends upon the last value now!
+                            self.compile_jump_if(last_value, false, target_label)?;
+                            self.set_label(end_label);
+                        }
                     }
                 }
-            },
+            }
+            ast::ExpressionType::Unop {
+                op: ast::UnaryOperator::Not,
+                a,
+            } => {
+                self.compile_jump_if(a, !condition, target_label)?;
+            }
             _ => {
+                // Fall back case which always will work!
                 self.compile_expression(expression)?;
-                match context {
-                    EvalContext::Statement => {
-                        if let Some(true_label) = true_label {
-                            self.emit(Instruction::JumpIf { target: true_label });
-                        }
-                        if let Some(false_label) = false_label {
-                            self.emit(Instruction::JumpIfFalse {
-                                target: false_label,
-                            });
-                        }
-                    }
-                    EvalContext::Expression => {
-                        if let Some(true_label) = true_label {
-                            self.emit(Instruction::Duplicate);
-                            self.emit(Instruction::JumpIf { target: true_label });
-                            self.emit(Instruction::Pop);
-                        }
-                        if let Some(false_label) = false_label {
-                            self.emit(Instruction::Duplicate);
-                            self.emit(Instruction::JumpIfFalse {
-                                target: false_label,
-                            });
-                            self.emit(Instruction::Pop);
-                        }
-                    }
+                if condition {
+                    self.emit(Instruction::JumpIfTrue {
+                        target: target_label,
+                    });
+                } else {
+                    self.emit(Instruction::JumpIfFalse {
+                        target: target_label,
+                    });
                 }
             }
         }
         Ok(())
     }
 
+    /// Compile a boolean operation as an expression.
+    /// This means, that the last value remains on the stack.
+    fn compile_bool_op(
+        &mut self,
+        op: &ast::BooleanOperator,
+        values: &[ast::Expression],
+    ) -> Result<(), CompileError> {
+        let end_label = self.new_label();
+
+        let (last_value, values) = values.split_last().unwrap();
+        for value in values {
+            self.compile_expression(value)?;
+
+            match op {
+                ast::BooleanOperator::And => {
+                    self.emit(Instruction::JumpIfFalseOrPop { target: end_label });
+                }
+                ast::BooleanOperator::Or => {
+                    self.emit(Instruction::JumpIfTrueOrPop { target: end_label });
+                }
+            }
+        }
+
+        // If all values did not qualify, take the value of the last value:
+        self.compile_expression(last_value)?;
+        self.set_label(end_label);
+        Ok(())
+    }
+
     fn compile_expression(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
         trace!("Compiling {:?}", expression);
         self.set_source_location(&expression.location);
@@ -1327,12 +1373,7 @@ impl<O: OutputStream> Compiler<O> {
                 args,
                 keywords,
             } => self.compile_call(function, args, keywords)?,
-            BoolOp { .. } => self.compile_test(
-                expression,
-                Option::None,
-                Option::None,
-                EvalContext::Expression,
-            )?,
+            BoolOp { op, values } => self.compile_bool_op(op, values)?,
             Binop { a, op, b } => {
                 self.compile_expression(a)?;
                 self.compile_expression(b)?;
@@ -1527,8 +1568,7 @@ impl<O: OutputStream> Compiler<O> {
             IfExpression { test, body, orelse } => {
                 let no_label = self.new_label();
                 let end_label = self.new_label();
-                self.compile_test(test, Option::None, Option::None, EvalContext::Expression)?;
-                self.emit(Instruction::JumpIfFalse { target: no_label });
+                self.compile_jump_if(test, false, no_label)?;
                 // True case
                 self.compile_expression(body)?;
                 self.emit(Instruction::Jump { target: end_label });
@@ -1745,12 +1785,7 @@ impl<O: OutputStream> Compiler<O> {
 
             // Now evaluate the ifs:
             for if_condition in &generator.ifs {
-                self.compile_test(
-                    if_condition,
-                    None,
-                    Some(start_label),
-                    EvalContext::Statement,
-                )?
+                self.compile_jump_if(if_condition, false, start_label)?
             }
         }
 
@@ -1988,11 +2023,11 @@ mod tests {
                 LoadConst {
                     value: Boolean { value: true }
                 },
-                JumpIf { target: 1 },
+                JumpIfTrue { target: 1 },
                 LoadConst {
                     value: Boolean { value: false }
                 },
-                JumpIf { target: 1 },
+                JumpIfTrue { target: 1 },
                 LoadConst {
                     value: Boolean { value: false }
                 },
@@ -2042,7 +2077,7 @@ mod tests {
                 LoadConst {
                     value: Boolean { value: false }
                 },
-                JumpIf { target: 1 },
+                JumpIfTrue { target: 1 },
                 LoadConst {
                     value: Boolean { value: false }
                 },
diff --git a/src/symboltable.rs b/src/symboltable.rs
index c1db24bf..3792bf4f 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -404,9 +404,8 @@ impl SymbolTableBuilder {
                 self.scan_expression(a)?;
                 self.scan_expression(b)?;
             }
-            BoolOp { a, b, .. } => {
-                self.scan_expression(a)?;
-                self.scan_expression(b)?;
+            BoolOp { values, .. } => {
+                self.scan_expressions(values)?;
             }
             Compare { vals, .. } => {
                 self.scan_expressions(vals)?;

From 5135c696e7510097c396e8fe45ec35a7cf21ded2 Mon Sep 17 00:00:00 2001
From: Dong-hee Na <donghee.na92@gmail.com>
Date: Mon, 12 Aug 2019 00:32:57 +0900
Subject: [PATCH 043/459] Fix scope_for_name to catch NameError properly

---
 src/compile.rs     | 4 +++-
 src/symboltable.rs | 6 ++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index a13d3617..74deb2d2 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1877,7 +1877,9 @@ impl<O: OutputStream> Compiler<O> {
     fn lookup_name(&self, name: &str) -> &Symbol {
         // println!("Looking up {:?}", name);
         let scope = self.scope_stack.last().unwrap();
-        scope.lookup(name).unwrap()
+        scope.lookup(name).expect(
+            "The symbol must be present in the symbol table, even when it is undefined in python.",
+        )
     }
 
     // Low level helper functions:
diff --git a/src/symboltable.rs b/src/symboltable.rs
index c1db24bf..6dc9c4c7 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -244,13 +244,11 @@ impl SymbolTableBuilder {
             } => {
                 self.scan_expressions(decorator_list)?;
                 self.register_name(name, SymbolRole::Assigned)?;
-
-                self.enter_function(args)?;
-
-                self.scan_statements(body)?;
                 if let Some(expression) = returns {
                     self.scan_expression(expression)?;
                 }
+                self.enter_function(args)?;
+                self.scan_statements(body)?;
                 self.leave_scope();
             }
             ClassDef {

From 11f4dc24dc18e92120fb4f587e3fb020b7010878 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sun, 11 Aug 2019 15:16:42 -0500
Subject: [PATCH 044/459] Change compile::compile() to take a Mode instead of
 an &Mode

---
 src/compile.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index 269e493b..f6f0e1cb 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -31,7 +31,7 @@ struct Compiler<O: OutputStream = BasicOutputStream> {
 /// Compile a given sourcecode into a bytecode object.
 pub fn compile(
     source: &str,
-    mode: &Mode,
+    mode: Mode,
     source_path: String,
     optimize: u8,
 ) -> Result<CodeObject, CompileError> {
@@ -102,6 +102,7 @@ pub fn compile_program_single(
     })
 }
 
+#[derive(Clone, Copy)]
 pub enum Mode {
     Exec,
     Eval,

From 240c3c417cf69218438fc85cf06fdb204844006f Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sun, 11 Aug 2019 22:15:07 -0500
Subject: [PATCH 045/459] impl std::str::FromStr for compile::Mode

---
 src/compile.rs | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/src/compile.rs b/src/compile.rs
index f6f0e1cb..9b0ad46f 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -109,6 +109,35 @@ pub enum Mode {
     Single,
 }
 
+impl std::str::FromStr for Mode {
+    type Err = ModeParseError;
+    fn from_str(s: &str) -> Result<Self, ModeParseError> {
+        match s {
+            "exec" => Ok(Mode::Exec),
+            "eval" => Ok(Mode::Eval),
+            "single" => Ok(Mode::Single),
+            _ => Err(ModeParseError { _priv: () }),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ModeParseError {
+    _priv: (),
+}
+
+impl std::fmt::Display for ModeParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, r#"mode should be "exec", "eval", or "single""#)
+    }
+}
+
+#[derive(Clone, Copy)]
+enum EvalContext {
+    Statement,
+    Expression,
+}
+
 pub(crate) type Label = usize;
 
 impl<O> Default for Compiler<O>

From 3b876205e5add94a3e48c84ea78a7afcce3b7a0c Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Tue, 13 Aug 2019 21:13:16 +0200
Subject: [PATCH 046/459] Improve symbol table processing.

---
 src/compile.rs     |  52 +++---
 src/symboltable.rs | 441 ++++++++++++++++++++++++++++-----------------
 2 files changed, 306 insertions(+), 187 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 269e493b..d854c0f4 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -8,7 +8,9 @@
 use crate::error::{CompileError, CompileErrorType};
 use crate::output_stream::{CodeObjectStream, OutputStream};
 use crate::peephole::PeepholeOptimizer;
-use crate::symboltable::{make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope};
+use crate::symboltable::{
+    make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope, SymbolTable,
+};
 use num_complex::Complex64;
 use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Varargs};
 use rustpython_parser::{ast, parser};
@@ -18,7 +20,7 @@ type BasicOutputStream = PeepholeOptimizer<CodeObjectStream>;
 /// Main structure holding the state of compilation.
 struct Compiler<O: OutputStream = BasicOutputStream> {
     output_stack: Vec<O>,
-    scope_stack: Vec<SymbolScope>,
+    symbol_table_stack: Vec<SymbolTable>,
     nxt_label: usize,
     source_path: Option<String>,
     current_source_location: ast::Location,
@@ -123,7 +125,7 @@ impl<O: OutputStream> Compiler<O> {
     fn new(optimize: u8) -> Self {
         Compiler {
             output_stack: Vec::new(),
-            scope_stack: Vec::new(),
+            symbol_table_stack: Vec::new(),
             nxt_label: 0,
             source_path: None,
             current_source_location: ast::Location::default(),
@@ -158,10 +160,10 @@ impl<O: OutputStream> Compiler<O> {
     fn compile_program(
         &mut self,
         program: &ast::Program,
-        symbol_scope: SymbolScope,
+        symbol_table: SymbolTable,
     ) -> Result<(), CompileError> {
         let size_before = self.output_stack.len();
-        self.scope_stack.push(symbol_scope);
+        self.symbol_table_stack.push(symbol_table);
         self.compile_statements(&program.statements)?;
         assert_eq!(self.output_stack.len(), size_before);
 
@@ -176,9 +178,9 @@ impl<O: OutputStream> Compiler<O> {
     fn compile_program_single(
         &mut self,
         program: &ast::Program,
-        symbol_scope: SymbolScope,
+        symbol_table: SymbolTable,
     ) -> Result<(), CompileError> {
-        self.scope_stack.push(symbol_scope);
+        self.symbol_table_stack.push(symbol_table);
 
         let mut emitted_return = false;
 
@@ -215,9 +217,9 @@ impl<O: OutputStream> Compiler<O> {
     fn compile_statement_eval(
         &mut self,
         statements: &[ast::Statement],
-        symbol_table: SymbolScope,
+        symbol_table: SymbolTable,
     ) -> Result<(), CompileError> {
-        self.scope_stack.push(symbol_table);
+        self.symbol_table_stack.push(symbol_table);
         for statement in statements {
             if let ast::StatementType::Expression { ref expression } = statement.node {
                 self.compile_expression(expression)?;
@@ -241,12 +243,11 @@ impl<O: OutputStream> Compiler<O> {
 
     fn scope_for_name(&self, name: &str) -> bytecode::NameScope {
         let symbol = self.lookup_name(name);
-        if symbol.is_global {
-            bytecode::NameScope::Global
-        } else if symbol.is_nonlocal {
-            bytecode::NameScope::NonLocal
-        } else {
-            bytecode::NameScope::Local
+        match symbol.scope {
+            SymbolScope::Global => bytecode::NameScope::Global,
+            SymbolScope::Nonlocal => bytecode::NameScope::NonLocal,
+            SymbolScope::Unknown => bytecode::NameScope::Local,
+            SymbolScope::Local => bytecode::NameScope::Local,
         }
     }
 
@@ -1897,22 +1898,27 @@ impl<O: OutputStream> Compiler<O> {
 
     // Scope helpers:
     fn enter_scope(&mut self) {
-        // println!("Enter scope {:?}", self.scope_stack);
+        // println!("Enter scope {:?}", self.symbol_table_stack);
         // Enter first subscope!
-        let scope = self.scope_stack.last_mut().unwrap().sub_scopes.remove(0);
-        self.scope_stack.push(scope);
+        let table = self
+            .symbol_table_stack
+            .last_mut()
+            .unwrap()
+            .sub_tables
+            .remove(0);
+        self.symbol_table_stack.push(table);
     }
 
     fn leave_scope(&mut self) {
-        // println!("Leave scope {:?}", self.scope_stack);
-        let scope = self.scope_stack.pop().unwrap();
-        assert!(scope.sub_scopes.is_empty());
+        // println!("Leave scope {:?}", self.symbol_table_stack);
+        let table = self.symbol_table_stack.pop().unwrap();
+        assert!(table.sub_tables.is_empty());
     }
 
     fn lookup_name(&self, name: &str) -> &Symbol {
         // println!("Looking up {:?}", name);
-        let scope = self.scope_stack.last().unwrap();
-        scope.lookup(name).expect(
+        let symbol_table = self.symbol_table_stack.last().unwrap();
+        symbol_table.lookup(name).expect(
             "The symbol must be present in the symbol table, even when it is undefined in python.",
         )
     }
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 20d602d6..d91f32dc 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -12,47 +12,59 @@ use indexmap::map::IndexMap;
 use rustpython_parser::ast;
 use rustpython_parser::location::Location;
 
-pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolScope, SymbolTableError> {
+pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolTable, SymbolTableError> {
     let mut builder: SymbolTableBuilder = Default::default();
-    builder.enter_scope();
+    builder.prepare();
     builder.scan_program(program)?;
-    assert_eq!(builder.scopes.len(), 1);
-
-    let symbol_table = builder.scopes.pop().unwrap();
-    analyze_symbol_table(&symbol_table, None)?;
-    Ok(symbol_table)
+    builder.finish()
 }
 
 pub fn statements_to_symbol_table(
     statements: &[ast::Statement],
-) -> Result<SymbolScope, SymbolTableError> {
+) -> Result<SymbolTable, SymbolTableError> {
     let mut builder: SymbolTableBuilder = Default::default();
-    builder.enter_scope();
+    builder.prepare();
     builder.scan_statements(statements)?;
-    assert_eq!(builder.scopes.len(), 1);
-
-    let symbol_table = builder.scopes.pop().unwrap();
-    analyze_symbol_table(&symbol_table, None)?;
-    Ok(symbol_table)
+    builder.finish()
 }
 
 /// Captures all symbols in the current scope, and has a list of subscopes in this scope.
 #[derive(Clone, Default)]
-pub struct SymbolScope {
+pub struct SymbolTable {
     /// A set of symbols present on this scope level.
     pub symbols: IndexMap<String, Symbol>,
 
     /// A list of subscopes in the order as found in the
     /// AST nodes.
-    pub sub_scopes: Vec<SymbolScope>,
+    pub sub_tables: Vec<SymbolTable>,
+}
+
+impl SymbolTable {
+    fn new() -> Self {
+        SymbolTable {
+            symbols: Default::default(),
+            sub_tables: vec![],
+        }
+    }
+}
+
+/// Indicator for a single symbol what the scope of this symbol is.
+/// The scope can be unknown, which is unfortunate, but not impossible.
+#[derive(Debug, Clone)]
+pub enum SymbolScope {
+    Global,
+    Nonlocal,
+    Local,
+    Unknown,
 }
 
+/// A single symbol in a table. Has various properties such as the scope
+/// of the symbol, and also the various uses of the symbol.
 #[derive(Debug, Clone)]
 pub struct Symbol {
     pub name: String,
-    pub is_global: bool,
-    pub is_local: bool,
-    pub is_nonlocal: bool,
+    // pub table: SymbolTableRef,
+    pub scope: SymbolScope,
     pub is_param: bool,
     pub is_referenced: bool,
     pub is_assigned: bool,
@@ -64,9 +76,8 @@ impl Symbol {
     fn new(name: &str) -> Self {
         Symbol {
             name: name.to_string(),
-            is_global: false,
-            is_local: false,
-            is_nonlocal: false,
+            // table,
+            scope: SymbolScope::Unknown,
             is_param: false,
             is_referenced: false,
             is_assigned: false,
@@ -74,6 +85,22 @@ impl Symbol {
             is_free: false,
         }
     }
+
+    pub fn is_global(&self) -> bool {
+        if let SymbolScope::Global = self.scope {
+            true
+        } else {
+            false
+        }
+    }
+
+    pub fn is_local(&self) -> bool {
+        if let SymbolScope::Local = self.scope {
+            true
+        } else {
+            false
+        }
+    }
 }
 
 #[derive(Debug)]
@@ -93,19 +120,19 @@ impl From<SymbolTableError> for CompileError {
 
 type SymbolTableResult = Result<(), SymbolTableError>;
 
-impl SymbolScope {
+impl SymbolTable {
     pub fn lookup(&self, name: &str) -> Option<&Symbol> {
         self.symbols.get(name)
     }
 }
 
-impl std::fmt::Debug for SymbolScope {
+impl std::fmt::Debug for SymbolTable {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(
             f,
-            "SymbolScope({:?} symbols, {:?} sub scopes)",
+            "SymbolTable({:?} symbols, {:?} sub scopes)",
             self.symbols.len(),
-            self.sub_scopes.len()
+            self.sub_tables.len()
         )
     }
 }
@@ -113,75 +140,127 @@ impl std::fmt::Debug for SymbolScope {
 /* Perform some sort of analysis on nonlocals, globals etc..
   See also: https://github.com/python/cpython/blob/master/Python/symtable.c#L410
 */
-fn analyze_symbol_table(
-    symbol_scope: &SymbolScope,
-    parent_symbol_scope: Option<&SymbolScope>,
-) -> SymbolTableResult {
-    // Analyze sub scopes:
-    for sub_scope in &symbol_scope.sub_scopes {
-        analyze_symbol_table(&sub_scope, Some(symbol_scope))?;
-    }
-
-    // Analyze symbols:
-    for symbol in symbol_scope.symbols.values() {
-        analyze_symbol(symbol, parent_symbol_scope)?;
-    }
+fn analyze_symbol_table(symbol_table: &mut SymbolTable) -> SymbolTableResult {
+    let mut analyzer = SymbolTableAnalyzer::default();
+    analyzer.analyze_symbol_table(symbol_table)
+}
 
-    Ok(())
+/// Symbol table analysis. Can be used to analyze a fully
+/// build symbol table structure. It will mark variables
+/// as local variables for example.
+#[derive(Default)]
+struct SymbolTableAnalyzer {
+    tables: Vec<SymbolTable>,
 }
 
-fn analyze_symbol(symbol: &Symbol, parent_symbol_scope: Option<&SymbolScope>) -> SymbolTableResult {
-    if symbol.is_nonlocal {
-        // check if name is defined in parent scope!
-        if let Some(parent_symbol_scope) = parent_symbol_scope {
-            if !parent_symbol_scope.symbols.contains_key(&symbol.name) {
-                return Err(SymbolTableError {
-                    error: format!("no binding for nonlocal '{}' found", symbol.name),
-                    location: Default::default(),
-                });
-            }
-        } else {
-            return Err(SymbolTableError {
-                error: format!(
-                    "nonlocal {} defined at place without an enclosing scope",
-                    symbol.name
-                ),
-                location: Default::default(),
-            });
+impl SymbolTableAnalyzer {
+    fn analyze_symbol_table(&mut self, symbol_table: &mut SymbolTable) -> SymbolTableResult {
+        // Store a copy to determine the parent.
+        // TODO: this should be improved to resolve this clone action.
+        self.tables.push(symbol_table.clone());
+
+        // Analyze sub scopes:
+        for sub_table in &mut symbol_table.sub_tables {
+            self.analyze_symbol_table(sub_table)?;
         }
-    }
+        self.tables.pop();
 
-    // TODO: add more checks for globals
+        // Analyze symbols:
+        for symbol in symbol_table.symbols.values_mut() {
+            self.analyze_symbol(symbol)?;
+        }
+
+        Ok(())
+    }
 
-    Ok(())
+    fn analyze_symbol(&self, symbol: &mut Symbol) -> SymbolTableResult {
+        match symbol.scope {
+            SymbolScope::Nonlocal => {
+                // check if name is defined in parent table!
+                let parent_symbol_table: Option<&SymbolTable> = self.tables.last();
+                // symbol.table.borrow().parent.clone();
+
+                if let Some(table) = parent_symbol_table {
+                    if !table.symbols.contains_key(&symbol.name) {
+                        return Err(SymbolTableError {
+                            error: format!("no binding for nonlocal '{}' found", symbol.name),
+                            location: Default::default(),
+                        });
+                    }
+                } else {
+                    return Err(SymbolTableError {
+                        error: format!(
+                            "nonlocal {} defined at place without an enclosing scope",
+                            symbol.name
+                        ),
+                        location: Default::default(),
+                    });
+                }
+            }
+            SymbolScope::Global => {
+                // TODO: add more checks for globals?
+            }
+            SymbolScope::Local => {
+                // all is well
+            }
+            SymbolScope::Unknown => {
+                if symbol.is_assigned {
+                    symbol.scope = SymbolScope::Local;
+                }
+            }
+        }
+        Ok(())
+    }
 }
 
 #[derive(Debug, Clone)]
-enum SymbolRole {
+enum SymbolUsage {
     Global,
     Nonlocal,
     Used,
     Assigned,
+    Parameter,
 }
 
 #[derive(Default)]
 struct SymbolTableBuilder {
     // Scope stack.
-    scopes: Vec<SymbolScope>,
+    tables: Vec<SymbolTable>,
+}
+
+/// Enum to indicate in what mode an expression
+/// was used.
+/// In cpython this is stored in the AST, but I think this
+/// is not logical, since it is not context free.
+enum ExpressionContext {
+    Load,
+    Store,
+    Delete,
 }
 
 impl SymbolTableBuilder {
-    fn enter_scope(&mut self) {
-        let scope = Default::default();
-        self.scopes.push(scope);
-        // self.work_scopes.push(Default::default());
+    fn prepare(&mut self) {
+        let table = SymbolTable::new();
+        self.tables.push(table);
+    }
+
+    fn finish(&mut self) -> Result<SymbolTable, SymbolTableError> {
+        assert_eq!(self.tables.len(), 1);
+        let mut symbol_table = self.tables.pop().unwrap();
+        analyze_symbol_table(&mut symbol_table)?;
+        Ok(symbol_table)
     }
 
-    fn leave_scope(&mut self) {
-        // Pop scope and add to subscopes of parent scope.
-        // let work_scope = self.work_scopes.pop().unwrap();
-        let scope = self.scopes.pop().unwrap();
-        self.scopes.last_mut().unwrap().sub_scopes.push(scope);
+    fn enter_block(&mut self) {
+        // let parent = Some(self.tables.last().unwrap().clone());
+        let table = SymbolTable::new();
+        self.tables.push(table);
+    }
+
+    fn leave_block(&mut self) {
+        // Pop symbol table and add to sub table of parent table.
+        let table = self.tables.pop().unwrap();
+        self.tables.last_mut().unwrap().sub_tables.push(table);
     }
 
     fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult {
@@ -204,7 +283,7 @@ impl SymbolTableBuilder {
     }
 
     fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
-        self.register_name(&parameter.arg, SymbolRole::Assigned)
+        self.register_name(&parameter.arg, SymbolUsage::Parameter)
     }
 
     fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
@@ -216,7 +295,7 @@ impl SymbolTableBuilder {
 
     fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
         if let Some(annotation) = &parameter.annotation {
-            self.scan_expression(&annotation)?;
+            self.scan_expression(&annotation, &ExpressionContext::Load)?;
         }
         Ok(())
     }
@@ -226,12 +305,12 @@ impl SymbolTableBuilder {
         match &statement.node {
             Global { names } => {
                 for name in names {
-                    self.register_name(name, SymbolRole::Global)?;
+                    self.register_name(name, SymbolUsage::Global)?;
                 }
             }
             Nonlocal { names } => {
                 for name in names {
-                    self.register_name(name, SymbolRole::Nonlocal)?;
+                    self.register_name(name, SymbolUsage::Nonlocal)?;
                 }
             }
             FunctionDef {
@@ -242,14 +321,14 @@ impl SymbolTableBuilder {
                 returns,
                 ..
             } => {
-                self.scan_expressions(decorator_list)?;
-                self.register_name(name, SymbolRole::Assigned)?;
+                self.scan_expressions(decorator_list, &ExpressionContext::Load)?;
+                self.register_name(name, SymbolUsage::Assigned)?;
                 if let Some(expression) = returns {
-                    self.scan_expression(expression)?;
+                    self.scan_expression(expression, &ExpressionContext::Load)?;
                 }
                 self.enter_function(args)?;
                 self.scan_statements(body)?;
-                self.leave_scope();
+                self.leave_block();
             }
             ClassDef {
                 name,
@@ -258,19 +337,21 @@ impl SymbolTableBuilder {
                 keywords,
                 decorator_list,
             } => {
-                self.register_name(name, SymbolRole::Assigned)?;
-                self.enter_scope();
+                self.register_name(name, SymbolUsage::Assigned)?;
+                self.enter_block();
                 self.scan_statements(body)?;
-                self.leave_scope();
-                self.scan_expressions(bases)?;
+                self.leave_block();
+                self.scan_expressions(bases, &ExpressionContext::Load)?;
                 for keyword in keywords {
-                    self.scan_expression(&keyword.value)?;
+                    self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
                 }
-                self.scan_expressions(decorator_list)?;
+                self.scan_expressions(decorator_list, &ExpressionContext::Load)?;
+            }
+            Expression { expression } => {
+                self.scan_expression(expression, &ExpressionContext::Load)?
             }
-            Expression { expression } => self.scan_expression(expression)?,
             If { test, body, orelse } => {
-                self.scan_expression(test)?;
+                self.scan_expression(test, &ExpressionContext::Load)?;
                 self.scan_statements(body)?;
                 if let Some(code) = orelse {
                     self.scan_statements(code)?;
@@ -283,15 +364,15 @@ impl SymbolTableBuilder {
                 orelse,
                 ..
             } => {
-                self.scan_expression(target)?;
-                self.scan_expression(iter)?;
+                self.scan_expression(target, &ExpressionContext::Store)?;
+                self.scan_expression(iter, &ExpressionContext::Load)?;
                 self.scan_statements(body)?;
                 if let Some(code) = orelse {
                     self.scan_statements(code)?;
                 }
             }
             While { test, body, orelse } => {
-                self.scan_expression(test)?;
+                self.scan_expression(test, &ExpressionContext::Load)?;
                 self.scan_statements(body)?;
                 if let Some(code) = orelse {
                     self.scan_statements(code)?;
@@ -304,51 +385,51 @@ impl SymbolTableBuilder {
                 for name in names {
                     if let Some(alias) = &name.alias {
                         // `import mymodule as myalias`
-                        self.register_name(alias, SymbolRole::Assigned)?;
+                        self.register_name(alias, SymbolUsage::Assigned)?;
                     } else {
                         // `import module`
-                        self.register_name(&name.symbol, SymbolRole::Assigned)?;
+                        self.register_name(&name.symbol, SymbolUsage::Assigned)?;
                     }
                 }
             }
             Return { value } => {
                 if let Some(expression) = value {
-                    self.scan_expression(expression)?;
+                    self.scan_expression(expression, &ExpressionContext::Load)?;
                 }
             }
             Assert { test, msg } => {
-                self.scan_expression(test)?;
+                self.scan_expression(test, &ExpressionContext::Load)?;
                 if let Some(expression) = msg {
-                    self.scan_expression(expression)?;
+                    self.scan_expression(expression, &ExpressionContext::Load)?;
                 }
             }
             Delete { targets } => {
-                self.scan_expressions(targets)?;
+                self.scan_expressions(targets, &ExpressionContext::Delete)?;
             }
             Assign { targets, value } => {
-                self.scan_expressions(targets)?;
-                self.scan_expression(value)?;
+                self.scan_expressions(targets, &ExpressionContext::Store)?;
+                self.scan_expression(value, &ExpressionContext::Load)?;
             }
             AugAssign { target, value, .. } => {
-                self.scan_expression(target)?;
-                self.scan_expression(value)?;
+                self.scan_expression(target, &ExpressionContext::Store)?;
+                self.scan_expression(value, &ExpressionContext::Load)?;
             }
             AnnAssign {
                 target,
                 annotation,
                 value,
             } => {
-                self.scan_expression(target)?;
-                self.scan_expression(annotation)?;
+                self.scan_expression(target, &ExpressionContext::Store)?;
+                self.scan_expression(annotation, &ExpressionContext::Load)?;
                 if let Some(value) = value {
-                    self.scan_expression(value)?;
+                    self.scan_expression(value, &ExpressionContext::Load)?;
                 }
             }
             With { items, body, .. } => {
                 for item in items {
-                    self.scan_expression(&item.context_expr)?;
+                    self.scan_expression(&item.context_expr, &ExpressionContext::Load)?;
                     if let Some(expression) = &item.optional_vars {
-                        self.scan_expression(expression)?;
+                        self.scan_expression(expression, &ExpressionContext::Store)?;
                     }
                 }
                 self.scan_statements(body)?;
@@ -362,10 +443,10 @@ impl SymbolTableBuilder {
                 self.scan_statements(body)?;
                 for handler in handlers {
                     if let Some(expression) = &handler.typ {
-                        self.scan_expression(expression)?;
+                        self.scan_expression(expression, &ExpressionContext::Load)?;
                     }
                     if let Some(name) = &handler.name {
-                        self.register_name(name, SymbolRole::Assigned)?;
+                        self.register_name(name, SymbolUsage::Assigned)?;
                     }
                     self.scan_statements(&handler.body)?;
                 }
@@ -378,94 +459,102 @@ impl SymbolTableBuilder {
             }
             Raise { exception, cause } => {
                 if let Some(expression) = exception {
-                    self.scan_expression(expression)?;
+                    self.scan_expression(expression, &ExpressionContext::Load)?;
                 }
                 if let Some(expression) = cause {
-                    self.scan_expression(expression)?;
+                    self.scan_expression(expression, &ExpressionContext::Load)?;
                 }
             }
         }
         Ok(())
     }
 
-    fn scan_expressions(&mut self, expressions: &[ast::Expression]) -> SymbolTableResult {
+    fn scan_expressions(
+        &mut self,
+        expressions: &[ast::Expression],
+        context: &ExpressionContext,
+    ) -> SymbolTableResult {
         for expression in expressions {
-            self.scan_expression(expression)?;
+            self.scan_expression(expression, context)?;
         }
         Ok(())
     }
 
-    fn scan_expression(&mut self, expression: &ast::Expression) -> SymbolTableResult {
+    fn scan_expression(
+        &mut self,
+        expression: &ast::Expression,
+        context: &ExpressionContext,
+    ) -> SymbolTableResult {
         use ast::ExpressionType::*;
         match &expression.node {
             Binop { a, b, .. } => {
-                self.scan_expression(a)?;
-                self.scan_expression(b)?;
+                self.scan_expression(a, context)?;
+                self.scan_expression(b, context)?;
             }
             BoolOp { values, .. } => {
-                self.scan_expressions(values)?;
+                self.scan_expressions(values, context)?;
             }
             Compare { vals, .. } => {
-                self.scan_expressions(vals)?;
+                self.scan_expressions(vals, context)?;
             }
             Subscript { a, b } => {
-                self.scan_expression(a)?;
-                self.scan_expression(b)?;
+                self.scan_expression(a, context)?;
+                self.scan_expression(b, context)?;
             }
             Attribute { value, .. } => {
-                self.scan_expression(value)?;
+                self.scan_expression(value, context)?;
             }
             Dict { elements } => {
                 for (key, value) in elements {
                     if let Some(key) = key {
-                        self.scan_expression(key)?;
+                        self.scan_expression(key, context)?;
                     } else {
                         // dict unpacking marker
                     }
-                    self.scan_expression(value)?;
+                    self.scan_expression(value, context)?;
                 }
             }
             Await { value } => {
-                self.scan_expression(value)?;
+                self.scan_expression(value, context)?;
             }
             Yield { value } => {
                 if let Some(expression) = value {
-                    self.scan_expression(expression)?;
+                    self.scan_expression(expression, context)?;
                 }
             }
             YieldFrom { value } => {
-                self.scan_expression(value)?;
+                self.scan_expression(value, context)?;
             }
             Unop { a, .. } => {
-                self.scan_expression(a)?;
+                self.scan_expression(a, context)?;
             }
             True | False | None | Ellipsis => {}
             Number { .. } => {}
             Starred { value } => {
-                self.scan_expression(value)?;
+                self.scan_expression(value, context)?;
             }
             Bytes { .. } => {}
             Tuple { elements } | Set { elements } | List { elements } | Slice { elements } => {
-                self.scan_expressions(elements)?;
+                self.scan_expressions(elements, &ExpressionContext::Load)?;
             }
             Comprehension { kind, generators } => {
                 match **kind {
                     ast::ComprehensionKind::GeneratorExpression { ref element }
                     | ast::ComprehensionKind::List { ref element }
                     | ast::ComprehensionKind::Set { ref element } => {
-                        self.scan_expression(element)?;
+                        self.scan_expression(element, &ExpressionContext::Load)?;
                     }
                     ast::ComprehensionKind::Dict { ref key, ref value } => {
-                        self.scan_expression(&key)?;
-                        self.scan_expression(&value)?;
+                        self.scan_expression(&key, &ExpressionContext::Load)?;
+                        self.scan_expression(&value, &ExpressionContext::Load)?;
                     }
                 }
 
                 for generator in generators {
-                    self.scan_expression(&generator.target)?;
-                    self.scan_expression(&generator.iter)?;
+                    self.scan_expression(&generator.target, &ExpressionContext::Store)?;
+                    self.scan_expression(&generator.iter, &ExpressionContext::Load)?;
                     for if_expr in &generator.ifs {
-                        self.scan_expression(if_expr)?;
+                        self.scan_expression(if_expr, &ExpressionContext::Load)?;
                     }
                 }
             }
@@ -474,27 +563,38 @@ impl SymbolTableBuilder {
                 args,
                 keywords,
             } => {
-                self.scan_expression(function)?;
-                self.scan_expressions(args)?;
+                self.scan_expression(function, &ExpressionContext::Load)?;
+                self.scan_expressions(args, &ExpressionContext::Load)?;
                 for keyword in keywords {
-                    self.scan_expression(&keyword.value)?;
+                    self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
                 }
             }
             String { value } => {
                 self.scan_string_group(value)?;
             }
             Identifier { name } => {
-                self.register_name(name, SymbolRole::Used)?;
+                // Determine the contextual usage of this symbol:
+                match context {
+                    ExpressionContext::Delete => {
+                        self.register_name(name, SymbolUsage::Used)?;
+                    }
+                    ExpressionContext::Load => {
+                        self.register_name(name, SymbolUsage::Used)?;
+                    }
+                    ExpressionContext::Store => {
+                        self.register_name(name, SymbolUsage::Assigned)?;
+                    }
+                }
             }
             Lambda { args, body } => {
                 self.enter_function(args)?;
-                self.scan_expression(body)?;
-                self.leave_scope();
+                self.scan_expression(body, &ExpressionContext::Load)?;
+                self.leave_block();
             }
             IfExpression { test, body, orelse } => {
-                self.scan_expression(test)?;
-                self.scan_expression(body)?;
-                self.scan_expression(orelse)?;
+                self.scan_expression(test, &ExpressionContext::Load)?;
+                self.scan_expression(body, &ExpressionContext::Load)?;
+                self.scan_expression(orelse, &ExpressionContext::Load)?;
             }
         }
         Ok(())
@@ -502,10 +602,10 @@ impl SymbolTableBuilder {
 
     fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult {
         // Evaluate eventual default parameters:
-        self.scan_expressions(&args.defaults)?;
+        self.scan_expressions(&args.defaults, &ExpressionContext::Load)?;
         for kw_default in &args.kw_defaults {
             if let Some(expression) = kw_default {
-                self.scan_expression(&expression)?;
+                self.scan_expression(&expression, &ExpressionContext::Load)?;
             }
         }
 
@@ -519,7 +619,7 @@ impl SymbolTableBuilder {
             self.scan_parameter_annotation(name)?;
         }
 
-        self.enter_scope();
+        self.enter_block();
 
         // Fill scope with parameter names:
         self.scan_parameters(&args.args)?;
@@ -537,7 +637,7 @@ impl SymbolTableBuilder {
         match group {
             ast::StringGroup::Constant { .. } => {}
             ast::StringGroup::FormattedValue { value, .. } => {
-                self.scan_expression(value)?;
+                self.scan_expression(value, &ExpressionContext::Load)?;
             }
             ast::StringGroup::Joined { values } => {
                 for subgroup in values {
@@ -549,22 +649,22 @@ impl SymbolTableBuilder {
     }
 
     #[allow(clippy::single_match)]
-    fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult {
-        let scope_depth = self.scopes.len();
-        let current_scope = self.scopes.last_mut().unwrap();
+    fn register_name(&mut self, name: &str, role: SymbolUsage) -> SymbolTableResult {
+        let scope_depth = self.tables.len();
+        let table = self.tables.last_mut().unwrap();
         let location = Default::default();
 
         // Some checks:
-        if current_scope.symbols.contains_key(name) {
+        if table.symbols.contains_key(name) {
             // Role already set..
             match role {
-                SymbolRole::Global => {
+                SymbolUsage::Global => {
                     return Err(SymbolTableError {
                         error: format!("name '{}' is used prior to global declaration", name),
                         location,
                     })
                 }
-                SymbolRole::Nonlocal => {
+                SymbolUsage::Nonlocal => {
                     return Err(SymbolTableError {
                         error: format!("name '{}' is used prior to nonlocal declaration", name),
                         location,
@@ -578,7 +678,7 @@ impl SymbolTableBuilder {
 
         // Some more checks:
         match role {
-            SymbolRole::Nonlocal => {
+            SymbolUsage::Nonlocal => {
                 if scope_depth < 2 {
                     return Err(SymbolTableError {
                         error: format!("cannot define nonlocal '{}' at top level.", name),
@@ -592,25 +692,38 @@ impl SymbolTableBuilder {
         }
 
         // Insert symbol when required:
-        if !current_scope.symbols.contains_key(name) {
+        if !table.symbols.contains_key(name) {
             let symbol = Symbol::new(name);
-            current_scope.symbols.insert(name.to_string(), symbol);
+            table.symbols.insert(name.to_string(), symbol);
         }
 
         // Set proper flags on symbol:
-        let symbol = current_scope.symbols.get_mut(name).unwrap();
+        let symbol = table.symbols.get_mut(name).unwrap();
         match role {
-            SymbolRole::Nonlocal => {
-                symbol.is_nonlocal = true;
+            SymbolUsage::Nonlocal => {
+                if let SymbolScope::Unknown = symbol.scope {
+                    symbol.scope = SymbolScope::Nonlocal;
+                } else {
+                    return Err(SymbolTableError {
+                        error: format!("Symbol {} scope cannot be set to nonlocal, since its scope was already determined otherwise.", name),
+                        location,
+                    });
+                }
             }
-            SymbolRole::Assigned => {
+            SymbolUsage::Parameter | SymbolUsage::Assigned => {
                 symbol.is_assigned = true;
-                // symbol.is_local = true;
             }
-            SymbolRole::Global => {
-                symbol.is_global = true;
+            SymbolUsage::Global => {
+                if let SymbolScope::Unknown = symbol.scope {
+                    symbol.scope = SymbolScope::Global;
+                } else {
+                    return Err(SymbolTableError {
+                        error: format!("Symbol {} scope cannot be set to global, since its scope was already determined otherwise.", name),
+                        location,
+                    });
+                }
             }
-            SymbolRole::Used => {
+            SymbolUsage::Used => {
                 symbol.is_referenced = true;
             }
         }

From f4ee74dbd41c46521e1f01fb8c27bd4d1d58e9c0 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Tue, 13 Aug 2019 23:04:32 -0500
Subject: [PATCH 047/459] Fix errors

---
 src/compile.rs | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 9b0ad46f..386fd925 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -132,12 +132,6 @@ impl std::fmt::Display for ModeParseError {
     }
 }
 
-#[derive(Clone, Copy)]
-enum EvalContext {
-    Statement,
-    Expression,
-}
-
 pub(crate) type Label = usize;
 
 impl<O> Default for Compiler<O>

From e0c375577702b07e4c07a5a4306b0973cce1435b Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Wed, 14 Aug 2019 20:43:23 +0200
Subject: [PATCH 048/459] Extend AST python module. Add idea for scope
 detection.

---
 src/symboltable.rs | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index d91f32dc..56d10670 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -204,8 +204,25 @@ impl SymbolTableAnalyzer {
                 // all is well
             }
             SymbolScope::Unknown => {
-                if symbol.is_assigned {
+                // Try hard to figure out what the scope of this symbol is.
+
+                if symbol.is_assigned || symbol.is_parameter {
                     symbol.scope = SymbolScope::Local;
+                } else {
+                    // TODO: comment this out and make it work properly:
+                    /*
+                    let found_in_outer_scope = self
+                        .tables
+                        .iter()
+                        .any(|t| t.symbols.contains_key(&symbol.name));
+                    if found_in_outer_scope {
+                        // Symbol is in some outer scope.
+
+                    } else {
+                        // Well, it must be a global then :)
+                        // symbol.scope = SymbolScope::Global;
+                    }
+                    */
                 }
             }
         }
@@ -710,7 +727,10 @@ impl SymbolTableBuilder {
                     });
                 }
             }
-            SymbolUsage::Parameter | SymbolUsage::Assigned => {
+            SymbolUsage::Parameter => {
+                symbol.is_parameter = true;
+            }
+            SymbolUsage::Assigned => {
                 symbol.is_assigned = true;
             }
             SymbolUsage::Global => {

From 7aac5b634fe7b2b055dc3d6135ce2d8681df7a78 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Tue, 13 Aug 2019 21:28:09 -0500
Subject: [PATCH 049/459] Fix weirdness with import submodules

---
 src/compile.rs     | 55 ++++++++++++++++++++++++++++++++--------------
 src/symboltable.rs |  5 ++++-
 2 files changed, 43 insertions(+), 17 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index baa838fe..e57ce3e2 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -300,16 +300,11 @@ impl<O: OutputStream> Compiler<O> {
             Import { names } => {
                 // import a, b, c as d
                 for name in names {
-                    self.emit(Instruction::Import {
-                        name: Some(name.symbol.clone()),
-                        symbols: vec![],
-                        level: 0,
-                    });
-
+                    self.compile_import(Some(&name.symbol), vec![], 0, name.alias.is_some());
                     if let Some(alias) = &name.alias {
                         self.store_name(alias);
                     } else {
-                        self.store_name(&name.symbol);
+                        self.store_name(name.symbol.split('.').next().unwrap());
                     }
                 }
             }
@@ -322,21 +317,25 @@ impl<O: OutputStream> Compiler<O> {
 
                 if import_star {
                     // from .... import *
-                    self.emit(Instruction::ImportStar {
-                        name: module.clone(),
-                        level: *level,
-                    });
+                    self.compile_import(
+                        module.as_ref().map(String::as_str),
+                        vec!["*".to_owned()],
+                        *level,
+                        false,
+                    );
+                    self.emit(Instruction::ImportStar);
                 } else {
                     // from mod import a, b as c
                     // First, determine the fromlist (for import lib):
                     let from_list = names.iter().map(|n| n.symbol.clone()).collect();
 
                     // Load module once:
-                    self.emit(Instruction::Import {
-                        name: module.clone(),
-                        symbols: from_list,
-                        level: *level,
-                    });
+                    self.compile_import(
+                        module.as_ref().map(String::as_str),
+                        from_list,
+                        *level,
+                        false,
+                    );
 
                     for name in names {
                         // import symbol from module:
@@ -574,6 +573,30 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
+    fn compile_import(
+        &mut self,
+        name: Option<&str>,
+        symbols: Vec<String>,
+        level: usize,
+        get_final_module: bool,
+    ) {
+        self.emit(Instruction::Import {
+            name: name.map(ToOwned::to_owned),
+            symbols,
+            level,
+        });
+
+        if get_final_module {
+            if let Some(name) = name {
+                for part in name.split('.').skip(1) {
+                    self.emit(Instruction::LoadAttr {
+                        name: part.to_owned(),
+                    });
+                }
+            }
+        }
+    }
+
     fn compile_delete(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
         match &expression.node {
             ast::ExpressionType::Identifier { name } => {
diff --git a/src/symboltable.rs b/src/symboltable.rs
index d91f32dc..8e44bc75 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -388,7 +388,10 @@ impl SymbolTableBuilder {
                         self.register_name(alias, SymbolUsage::Assigned)?;
                     } else {
                         // `import module`
-                        self.register_name(&name.symbol, SymbolUsage::Assigned)?;
+                        self.register_name(
+                            name.symbol.split('.').next().unwrap(),
+                            SymbolUsage::Assigned,
+                        )?;
                     }
                 }
             }

From 623472c366c79432dd7405adfd8a7e76948cbe0d Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 14 Aug 2019 09:53:30 -0500
Subject: [PATCH 050/459] Address feedback, simplify compilation code

---
 src/compile.rs | 57 ++++++++++++++++++--------------------------------
 1 file changed, 20 insertions(+), 37 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index e57ce3e2..1daf80e5 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -300,8 +300,17 @@ impl<O: OutputStream> Compiler<O> {
             Import { names } => {
                 // import a, b, c as d
                 for name in names {
-                    self.compile_import(Some(&name.symbol), vec![], 0, name.alias.is_some());
+                    self.emit(Instruction::Import {
+                        name: Some(name.symbol.clone()),
+                        symbols: vec![],
+                        level: 0,
+                    });
                     if let Some(alias) = &name.alias {
+                        for part in name.symbol.split('.').skip(1) {
+                            self.emit(Instruction::LoadAttr {
+                                name: part.to_owned(),
+                            });
+                        }
                         self.store_name(alias);
                     } else {
                         self.store_name(name.symbol.split('.').next().unwrap());
@@ -317,12 +326,11 @@ impl<O: OutputStream> Compiler<O> {
 
                 if import_star {
                     // from .... import *
-                    self.compile_import(
-                        module.as_ref().map(String::as_str),
-                        vec!["*".to_owned()],
-                        *level,
-                        false,
-                    );
+                    self.emit(Instruction::Import {
+                        name: module.clone(),
+                        symbols: vec!["*".to_owned()],
+                        level: *level,
+                    });
                     self.emit(Instruction::ImportStar);
                 } else {
                     // from mod import a, b as c
@@ -330,12 +338,11 @@ impl<O: OutputStream> Compiler<O> {
                     let from_list = names.iter().map(|n| n.symbol.clone()).collect();
 
                     // Load module once:
-                    self.compile_import(
-                        module.as_ref().map(String::as_str),
-                        from_list,
-                        *level,
-                        false,
-                    );
+                    self.emit(Instruction::Import {
+                        name: module.clone(),
+                        symbols: from_list,
+                        level: *level,
+                    });
 
                     for name in names {
                         // import symbol from module:
@@ -573,30 +580,6 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
-    fn compile_import(
-        &mut self,
-        name: Option<&str>,
-        symbols: Vec<String>,
-        level: usize,
-        get_final_module: bool,
-    ) {
-        self.emit(Instruction::Import {
-            name: name.map(ToOwned::to_owned),
-            symbols,
-            level,
-        });
-
-        if get_final_module {
-            if let Some(name) = name {
-                for part in name.split('.').skip(1) {
-                    self.emit(Instruction::LoadAttr {
-                        name: part.to_owned(),
-                    });
-                }
-            }
-        }
-    }
-
     fn compile_delete(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
         match &expression.node {
             ast::ExpressionType::Identifier { name } => {

From 866defbcf3ec60fe793cbfd67f5ad2d3b74250ee Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Thu, 15 Aug 2019 22:25:30 -0500
Subject: [PATCH 051/459] Compile docstrings for modules

---
 src/compile.rs | 95 +++++++++++++++++++++++++++++++++++---------------
 1 file changed, 66 insertions(+), 29 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 1daf80e5..23b46f1b 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -188,7 +188,19 @@ impl<O: OutputStream> Compiler<O> {
     ) -> Result<(), CompileError> {
         let size_before = self.output_stack.len();
         self.symbol_table_stack.push(symbol_table);
-        self.compile_statements(&program.statements)?;
+
+        let (statements, doc) = get_doc(&program.statements);
+        if let Some(value) = doc {
+            self.emit(Instruction::LoadConst {
+                value: bytecode::Constant::String { value },
+            });
+            self.emit(Instruction::StoreName {
+                name: "__doc__".to_owned(),
+                scope: bytecode::NameScope::Global,
+            });
+        }
+        self.compile_statements(statements)?;
+
         assert_eq!(self.output_stack.len(), size_before);
 
         // Emit None at end:
@@ -1897,30 +1909,36 @@ impl<O: OutputStream> Compiler<O> {
     }
 
     fn compile_string(&mut self, string: &ast::StringGroup) -> Result<(), CompileError> {
-        match string {
-            ast::StringGroup::Joined { values } => {
-                for value in values {
-                    self.compile_string(value)?;
+        if let Some(value) = try_get_constant_string(string) {
+            self.emit(Instruction::LoadConst {
+                value: bytecode::Constant::String { value },
+            });
+        } else {
+            match string {
+                ast::StringGroup::Joined { values } => {
+                    for value in values {
+                        self.compile_string(value)?;
+                    }
+                    self.emit(Instruction::BuildString { size: values.len() })
+                }
+                ast::StringGroup::Constant { value } => {
+                    self.emit(Instruction::LoadConst {
+                        value: bytecode::Constant::String {
+                            value: value.to_string(),
+                        },
+                    });
+                }
+                ast::StringGroup::FormattedValue {
+                    value,
+                    conversion,
+                    spec,
+                } => {
+                    self.compile_expression(value)?;
+                    self.emit(Instruction::FormatValue {
+                        conversion: conversion.map(compile_conversion_flag),
+                        spec: spec.clone(),
+                    });
                 }
-                self.emit(Instruction::BuildString { size: values.len() })
-            }
-            ast::StringGroup::Constant { value } => {
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::String {
-                        value: value.to_string(),
-                    },
-                });
-            }
-            ast::StringGroup::FormattedValue {
-                value,
-                conversion,
-                spec,
-            } => {
-                self.compile_expression(value)?;
-                self.emit(Instruction::FormatValue {
-                    conversion: conversion.map(compile_conversion_flag),
-                    spec: spec.clone(),
-                });
             }
         }
         Ok(())
@@ -2000,13 +2018,11 @@ impl<O: OutputStream> Compiler<O> {
 }
 
 fn get_doc(body: &[ast::Statement]) -> (&[ast::Statement], Option<String>) {
-    if let Some(val) = body.get(0) {
+    if let Some((val, body_rest)) = body.split_first() {
         if let ast::StatementType::Expression { ref expression } = val.node {
             if let ast::ExpressionType::String { value } = &expression.node {
-                if let ast::StringGroup::Constant { ref value } = value {
-                    if let Some((_, body_rest)) = body.split_first() {
-                        return (body_rest, Some(value.to_string()));
-                    }
+                if let Some(value) = try_get_constant_string(value) {
+                    return (body_rest, Some(value.to_string()));
                 }
             }
         }
@@ -2014,6 +2030,27 @@ fn get_doc(body: &[ast::Statement]) -> (&[ast::Statement], Option<String>) {
     (body, None)
 }
 
+fn try_get_constant_string(string: &ast::StringGroup) -> Option<String> {
+    fn get_constant_string_inner(out_string: &mut String, string: &ast::StringGroup) -> bool {
+        match string {
+            ast::StringGroup::Constant { value } => {
+                out_string.push_str(&value);
+                true
+            }
+            ast::StringGroup::Joined { values } => values
+                .iter()
+                .all(|value| get_constant_string_inner(out_string, value)),
+            ast::StringGroup::FormattedValue { .. } => false,
+        }
+    }
+    let mut out_string = String::new();
+    if get_constant_string_inner(&mut out_string, string) {
+        Some(out_string)
+    } else {
+        None
+    }
+}
+
 fn compile_location(location: &ast::Location) -> bytecode::Location {
     bytecode::Location::new(location.row(), location.column())
 }

From 9b1771b4837e1eff4f5215bbe43b872632962cad Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Sun, 18 Aug 2019 10:03:00 +0200
Subject: [PATCH 052/459] Fix clippy warnings. Add proper names to
 symboltables.

---
 src/symboltable.rs | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 42dadee3..46174c3b 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -31,6 +31,9 @@ pub fn statements_to_symbol_table(
 /// Captures all symbols in the current scope, and has a list of subscopes in this scope.
 #[derive(Clone, Default)]
 pub struct SymbolTable {
+    /// The name of this symbol table. Often the name of the class or function.
+    pub name: String,
+
     /// A set of symbols present on this scope level.
     pub symbols: IndexMap<String, Symbol>,
 
@@ -40,8 +43,9 @@ pub struct SymbolTable {
 }
 
 impl SymbolTable {
-    fn new() -> Self {
+    fn new(name: String) -> Self {
         SymbolTable {
+            name,
             symbols: Default::default(),
             sub_tables: vec![],
         }
@@ -211,18 +215,20 @@ impl SymbolTableAnalyzer {
                 } else {
                     // TODO: comment this out and make it work properly:
                     /*
+                     */
                     let found_in_outer_scope = self
                         .tables
                         .iter()
+                        .skip(1)
                         .any(|t| t.symbols.contains_key(&symbol.name));
+
                     if found_in_outer_scope {
                         // Symbol is in some outer scope.
-
+                        symbol.is_free = true;
                     } else {
                         // Well, it must be a global then :)
                         // symbol.scope = SymbolScope::Global;
                     }
-                    */
                 }
             }
         }
@@ -257,8 +263,7 @@ enum ExpressionContext {
 
 impl SymbolTableBuilder {
     fn prepare(&mut self) {
-        let table = SymbolTable::new();
-        self.tables.push(table);
+        self.enter_block("top")
     }
 
     fn finish(&mut self) -> Result<SymbolTable, SymbolTableError> {
@@ -268,9 +273,9 @@ impl SymbolTableBuilder {
         Ok(symbol_table)
     }
 
-    fn enter_block(&mut self) {
+    fn enter_block(&mut self, name: &str) {
         // let parent = Some(self.tables.last().unwrap().clone());
-        let table = SymbolTable::new();
+        let table = SymbolTable::new(name.to_string());
         self.tables.push(table);
     }
 
@@ -343,7 +348,7 @@ impl SymbolTableBuilder {
                 if let Some(expression) = returns {
                     self.scan_expression(expression, &ExpressionContext::Load)?;
                 }
-                self.enter_function(args)?;
+                self.enter_function(name, args)?;
                 self.scan_statements(body)?;
                 self.leave_block();
             }
@@ -355,7 +360,7 @@ impl SymbolTableBuilder {
                 decorator_list,
             } => {
                 self.register_name(name, SymbolUsage::Assigned)?;
-                self.enter_block();
+                self.enter_block(name);
                 self.scan_statements(body)?;
                 self.leave_block();
                 self.scan_expressions(bases, &ExpressionContext::Load)?;
@@ -607,7 +612,7 @@ impl SymbolTableBuilder {
                 }
             }
             Lambda { args, body } => {
-                self.enter_function(args)?;
+                self.enter_function("lambda", args)?;
                 self.scan_expression(body, &ExpressionContext::Load)?;
                 self.leave_block();
             }
@@ -620,7 +625,7 @@ impl SymbolTableBuilder {
         Ok(())
     }
 
-    fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult {
+    fn enter_function(&mut self, name: &str, args: &ast::Parameters) -> SymbolTableResult {
         // Evaluate eventual default parameters:
         self.scan_expressions(&args.defaults, &ExpressionContext::Load)?;
         for kw_default in &args.kw_defaults {
@@ -639,7 +644,7 @@ impl SymbolTableBuilder {
             self.scan_parameter_annotation(name)?;
         }
 
-        self.enter_block();
+        self.enter_block(name);
 
         // Fill scope with parameter names:
         self.scan_parameters(&args.args)?;

From 39c095d12c6d6f17d3575126452a4f85c89a38e9 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Sun, 18 Aug 2019 12:12:27 +0200
Subject: [PATCH 053/459] Add Free enum variant to the symbol scope to be able
 to distuingish between true local and free variables.

---
 src/compile.rs     | 12 ++++++------
 src/symboltable.rs | 18 +++++++++---------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 23b46f1b..2a95190d 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -282,8 +282,8 @@ impl<O: OutputStream> Compiler<O> {
         match symbol.scope {
             SymbolScope::Global => bytecode::NameScope::Global,
             SymbolScope::Nonlocal => bytecode::NameScope::NonLocal,
-            SymbolScope::Unknown => bytecode::NameScope::Local,
-            SymbolScope::Local => bytecode::NameScope::Local,
+            SymbolScope::Unknown => bytecode::NameScope::Free,
+            SymbolScope::Local => bytecode::NameScope::Free,
         }
     }
 
@@ -500,7 +500,7 @@ impl<O: OutputStream> Compiler<O> {
                     self.compile_jump_if(test, true, end_label)?;
                     self.emit(Instruction::LoadName {
                         name: String::from("AssertionError"),
-                        scope: bytecode::NameScope::Local,
+                        scope: bytecode::NameScope::Global,
                     });
                     match msg {
                         Some(e) => {
@@ -736,7 +736,7 @@ impl<O: OutputStream> Compiler<O> {
                 // Check exception type:
                 self.emit(Instruction::LoadName {
                     name: String::from("isinstance"),
-                    scope: bytecode::NameScope::Local,
+                    scope: bytecode::NameScope::Global,
                 });
                 self.emit(Instruction::Rotate { amount: 2 });
                 self.compile_expression(exc_type)?;
@@ -931,11 +931,11 @@ impl<O: OutputStream> Compiler<O> {
 
         self.emit(Instruction::LoadName {
             name: "__name__".to_string(),
-            scope: bytecode::NameScope::Local,
+            scope: bytecode::NameScope::Free,
         });
         self.emit(Instruction::StoreName {
             name: "__module__".to_string(),
-            scope: bytecode::NameScope::Local,
+            scope: bytecode::NameScope::Free,
         });
         self.compile_statements(new_body)?;
         self.emit(Instruction::LoadConst {
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 46174c3b..d99ae821 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -213,21 +213,21 @@ impl SymbolTableAnalyzer {
                 if symbol.is_assigned || symbol.is_parameter {
                     symbol.scope = SymbolScope::Local;
                 } else {
-                    // TODO: comment this out and make it work properly:
-                    /*
-                     */
-                    let found_in_outer_scope = self
-                        .tables
-                        .iter()
-                        .skip(1)
-                        .any(|t| t.symbols.contains_key(&symbol.name));
+                    // Interesting stuff about the __class__ variable:
+                    // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
+                    let found_in_outer_scope = (symbol.name == "__class__")
+                        || self
+                            .tables
+                            .iter()
+                            .skip(1)
+                            .any(|t| t.symbols.contains_key(&symbol.name));
 
                     if found_in_outer_scope {
                         // Symbol is in some outer scope.
                         symbol.is_free = true;
                     } else {
                         // Well, it must be a global then :)
-                        // symbol.scope = SymbolScope::Global;
+                        symbol.scope = SymbolScope::Global;
                     }
                 }
             }

From e8db01a09ab5b171cf57718a171d550e32a05cb3 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Tue, 27 Aug 2019 19:40:00 +0200
Subject: [PATCH 054/459] Refactor block stack unwinding to enable finally
 execution.

---
 src/compile.rs | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 2a95190d..70bc8f2d 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -708,12 +708,20 @@ impl<O: OutputStream> Compiler<O> {
         &mut self,
         body: &[ast::Statement],
         handlers: &[ast::ExceptHandler],
-        orelse: &Option<Vec<ast::Statement>>,
-        finalbody: &Option<Vec<ast::Statement>>,
+        orelse: &Option<ast::Suite>,
+        finalbody: &Option<ast::Suite>,
     ) -> Result<(), CompileError> {
         let mut handler_label = self.new_label();
         let finally_label = self.new_label();
         let else_label = self.new_label();
+
+        // Setup a finally block if we have a finally statement.
+        if finalbody.is_some() {
+            self.emit(Instruction::SetupFinally {
+                handler: finally_label,
+            });
+        }
+
         // try:
         self.emit(Instruction::SetupExcept {
             handler: handler_label,
@@ -798,8 +806,9 @@ impl<O: OutputStream> Compiler<O> {
         self.set_label(finally_label);
         if let Some(statements) = finalbody {
             self.compile_statements(statements)?;
+            self.emit(Instruction::EndFinally);
         }
-        // unimplemented!();
+
         Ok(())
     }
 

From e37f117035f900e70e1a4a510c3e9e2c9d1d2f30 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Tue, 27 Aug 2019 21:16:59 +0200
Subject: [PATCH 055/459] Implement execution of finally block. Fixes #1306.

---
 src/compile.rs | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 70bc8f2d..40a901e3 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -712,13 +712,13 @@ impl<O: OutputStream> Compiler<O> {
         finalbody: &Option<ast::Suite>,
     ) -> Result<(), CompileError> {
         let mut handler_label = self.new_label();
-        let finally_label = self.new_label();
+        let finally_handler_label = self.new_label();
         let else_label = self.new_label();
 
         // Setup a finally block if we have a finally statement.
         if finalbody.is_some() {
             self.emit(Instruction::SetupFinally {
-                handler: finally_label,
+                handler: finally_handler_label,
             });
         }
 
@@ -773,26 +773,28 @@ impl<O: OutputStream> Compiler<O> {
             // Handler code:
             self.compile_statements(&handler.body)?;
             self.emit(Instruction::PopException);
+
+            if finalbody.is_some() {
+                self.emit(Instruction::PopBlock); // pop finally block
+                                                  // We enter the finally block, without exception.
+                self.emit(Instruction::EnterFinally);
+            }
+
             self.emit(Instruction::Jump {
-                target: finally_label,
+                target: finally_handler_label,
             });
 
             // Emit a new label for the next handler
             self.set_label(handler_label);
             handler_label = self.new_label();
         }
+
         self.emit(Instruction::Jump {
             target: handler_label,
         });
         self.set_label(handler_label);
         // If code flows here, we have an unhandled exception,
-        // emit finally code and raise again!
-        // Duplicate finally code here:
-        // TODO: this bytecode is now duplicate, could this be
-        // improved?
-        if let Some(statements) = finalbody {
-            self.compile_statements(statements)?;
-        }
+        // raise the exception again!
         self.emit(Instruction::Raise { argc: 0 });
 
         // We successfully ran the try block:
@@ -802,8 +804,15 @@ impl<O: OutputStream> Compiler<O> {
             self.compile_statements(statements)?;
         }
 
+        if finalbody.is_some() {
+            self.emit(Instruction::PopBlock); // pop finally block
+
+            // We enter the finally block, without return / exception.
+            self.emit(Instruction::EnterFinally);
+        }
+
         // finally:
-        self.set_label(finally_label);
+        self.set_label(finally_handler_label);
         if let Some(statements) = finalbody {
             self.compile_statements(statements)?;
             self.emit(Instruction::EndFinally);

From e0a52c15f6e51f65bf6c94f929a31353bba0b571 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Tue, 27 Aug 2019 21:18:09 +0000
Subject: [PATCH 056/459] Make bytecode::Label its own struct

---
 src/compile.rs       | 48 ++++++++++++++++++++++++++++++--------------
 src/output_stream.rs |  3 +--
 src/peephole.rs      |  5 ++---
 3 files changed, 36 insertions(+), 20 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 40a901e3..40173dba 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -12,7 +12,7 @@ use crate::symboltable::{
     make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope, SymbolTable,
 };
 use num_complex::Complex64;
-use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Varargs};
+use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Label, Varargs};
 use rustpython_parser::{ast, parser};
 
 type BasicOutputStream = PeepholeOptimizer<CodeObjectStream>;
@@ -134,8 +134,6 @@ impl std::fmt::Display for ModeParseError {
     }
 }
 
-pub(crate) type Label = usize;
-
 impl<O> Default for Compiler<O>
 where
     O: OutputStream,
@@ -2004,7 +2002,7 @@ impl<O: OutputStream> Compiler<O> {
 
     // Generate a new label
     fn new_label(&mut self) -> Label {
-        let l = self.nxt_label;
+        let l = Label::new(self.nxt_label);
         self.nxt_label += 1;
         l
     }
@@ -2093,9 +2091,9 @@ fn compile_conversion_flag(conversion_flag: ast::ConversionFlag) -> bytecode::Co
 mod tests {
     use super::Compiler;
     use crate::symboltable::make_symbol_table;
-    use rustpython_bytecode::bytecode::CodeObject;
     use rustpython_bytecode::bytecode::Constant::*;
     use rustpython_bytecode::bytecode::Instruction::*;
+    use rustpython_bytecode::bytecode::{CodeObject, Label};
     use rustpython_parser::parser;
 
     fn compile_exec(source: &str) -> CodeObject {
@@ -2116,15 +2114,21 @@ mod tests {
                 LoadConst {
                     value: Boolean { value: true }
                 },
-                JumpIfTrue { target: 1 },
+                JumpIfTrue {
+                    target: Label::new(1)
+                },
                 LoadConst {
                     value: Boolean { value: false }
                 },
-                JumpIfTrue { target: 1 },
+                JumpIfTrue {
+                    target: Label::new(1)
+                },
                 LoadConst {
                     value: Boolean { value: false }
                 },
-                JumpIfFalse { target: 0 },
+                JumpIfFalse {
+                    target: Label::new(0)
+                },
                 Pass,
                 LoadConst { value: None },
                 ReturnValue
@@ -2141,15 +2145,21 @@ mod tests {
                 LoadConst {
                     value: Boolean { value: true }
                 },
-                JumpIfFalse { target: 0 },
+                JumpIfFalse {
+                    target: Label::new(0)
+                },
                 LoadConst {
                     value: Boolean { value: false }
                 },
-                JumpIfFalse { target: 0 },
+                JumpIfFalse {
+                    target: Label::new(0)
+                },
                 LoadConst {
                     value: Boolean { value: false }
                 },
-                JumpIfFalse { target: 0 },
+                JumpIfFalse {
+                    target: Label::new(0)
+                },
                 Pass,
                 LoadConst { value: None },
                 ReturnValue
@@ -2166,19 +2176,27 @@ mod tests {
                 LoadConst {
                     value: Boolean { value: true }
                 },
-                JumpIfFalse { target: 2 },
+                JumpIfFalse {
+                    target: Label::new(2)
+                },
                 LoadConst {
                     value: Boolean { value: false }
                 },
-                JumpIfTrue { target: 1 },
+                JumpIfTrue {
+                    target: Label::new(1)
+                },
                 LoadConst {
                     value: Boolean { value: false }
                 },
-                JumpIfFalse { target: 0 },
+                JumpIfFalse {
+                    target: Label::new(0)
+                },
                 LoadConst {
                     value: Boolean { value: true }
                 },
-                JumpIfFalse { target: 0 },
+                JumpIfFalse {
+                    target: Label::new(0)
+                },
                 Pass,
                 LoadConst { value: None },
                 ReturnValue
diff --git a/src/output_stream.rs b/src/output_stream.rs
index 35321dc5..9d0ece09 100644
--- a/src/output_stream.rs
+++ b/src/output_stream.rs
@@ -1,5 +1,4 @@
-use crate::compile::Label;
-use rustpython_bytecode::bytecode::{CodeObject, Instruction, Location};
+use rustpython_bytecode::bytecode::{CodeObject, Instruction, Label, Location};
 
 pub trait OutputStream: From<CodeObject> + Into<CodeObject> {
     /// Output an instruction
diff --git a/src/peephole.rs b/src/peephole.rs
index 64c06b81..c67a16a8 100644
--- a/src/peephole.rs
+++ b/src/peephole.rs
@@ -1,7 +1,6 @@
-use crate::compile::Label;
 use crate::output_stream::OutputStream;
 use arrayvec::ArrayVec;
-use rustpython_bytecode::bytecode::{self, CodeObject, Instruction, Location};
+use rustpython_bytecode::bytecode::{self, CodeObject, Instruction, Label, Location};
 
 const PEEPHOLE_BUFFER_SIZE: usize = 20;
 
@@ -92,7 +91,7 @@ where
         self.push(instruction, loc.into());
         optimize(self);
     }
-    fn set_label(&mut self, label: crate::compile::Label) {
+    fn set_label(&mut self, label: Label) {
         if let Some(instr) = self.buffer.last_mut() {
             instr.1.labels.push(label)
         }

From dac20eeb59da00ac130df944326f73ad7fc91132 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Wed, 28 Aug 2019 12:06:46 +0200
Subject: [PATCH 057/459] Make subscript its own bytecode. Remove pass
 bytecode. Move complex bytecodes into seperate functions to reduce complexity
 of dispatch function.

---
 src/compile.rs | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 40173dba..7cdfa1a3 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -584,7 +584,7 @@ impl<O: OutputStream> Compiler<O> {
                 }
             }
             Pass => {
-                self.emit(Instruction::Pass);
+                // No need to emit any code here :)
             }
         }
         Ok(())
@@ -1443,10 +1443,7 @@ impl<O: OutputStream> Compiler<O> {
             Subscript { a, b } => {
                 self.compile_expression(a)?;
                 self.compile_expression(b)?;
-                self.emit(Instruction::BinaryOperation {
-                    op: bytecode::BinaryOperator::Subscript,
-                    inplace: false,
-                });
+                self.emit(Instruction::Subscript);
             }
             Unop { op, a } => {
                 self.compile_expression(a)?;
@@ -2129,7 +2126,6 @@ mod tests {
                 JumpIfFalse {
                     target: Label::new(0)
                 },
-                Pass,
                 LoadConst { value: None },
                 ReturnValue
             ],
@@ -2160,7 +2156,6 @@ mod tests {
                 JumpIfFalse {
                     target: Label::new(0)
                 },
-                Pass,
                 LoadConst { value: None },
                 ReturnValue
             ],
@@ -2197,7 +2192,6 @@ mod tests {
                 JumpIfFalse {
                     target: Label::new(0)
                 },
-                Pass,
                 LoadConst { value: None },
                 ReturnValue
             ],

From a0dcfdd57fade6c70ae7c688dcc092144ec90f38 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Wed, 28 Aug 2019 16:10:30 +0200
Subject: [PATCH 058/459] Add scope type and other symboltable properties.

---
 src/symboltable.rs | 62 ++++++++++++++++++++++++++++++++++------------
 1 file changed, 46 insertions(+), 16 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index d99ae821..3c70d4fc 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -11,6 +11,7 @@ use crate::error::{CompileError, CompileErrorType};
 use indexmap::map::IndexMap;
 use rustpython_parser::ast;
 use rustpython_parser::location::Location;
+use std::fmt;
 
 pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolTable, SymbolTableError> {
     let mut builder: SymbolTableBuilder = Default::default();
@@ -29,11 +30,17 @@ pub fn statements_to_symbol_table(
 }
 
 /// Captures all symbols in the current scope, and has a list of subscopes in this scope.
-#[derive(Clone, Default)]
+#[derive(Clone)]
 pub struct SymbolTable {
     /// The name of this symbol table. Often the name of the class or function.
     pub name: String,
 
+    /// The type of symbol table
+    pub typ: SymbolTableType,
+
+    /// The line number in the sourcecode where this symboltable begins.
+    pub line_number: usize,
+
     /// A set of symbols present on this scope level.
     pub symbols: IndexMap<String, Symbol>,
 
@@ -43,15 +50,34 @@ pub struct SymbolTable {
 }
 
 impl SymbolTable {
-    fn new(name: String) -> Self {
+    fn new(name: String, typ: SymbolTableType, line_number: usize) -> Self {
         SymbolTable {
             name,
+            typ,
+            line_number,
             symbols: Default::default(),
             sub_tables: vec![],
         }
     }
 }
 
+#[derive(Clone)]
+pub enum SymbolTableType {
+    Module,
+    Class,
+    Function,
+}
+
+impl fmt::Display for SymbolTableType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            SymbolTableType::Module => write!(f, "module"),
+            SymbolTableType::Class => write!(f, "class"),
+            SymbolTableType::Function => write!(f, "function"),
+        }
+    }
+}
+
 /// Indicator for a single symbol what the scope of this symbol is.
 /// The scope can be unknown, which is unfortunate, but not impossible.
 #[derive(Debug, Clone)]
@@ -263,7 +289,7 @@ enum ExpressionContext {
 
 impl SymbolTableBuilder {
     fn prepare(&mut self) {
-        self.enter_block("top")
+        self.enter_scope("top", SymbolTableType::Module, 0)
     }
 
     fn finish(&mut self) -> Result<SymbolTable, SymbolTableError> {
@@ -273,14 +299,13 @@ impl SymbolTableBuilder {
         Ok(symbol_table)
     }
 
-    fn enter_block(&mut self, name: &str) {
-        // let parent = Some(self.tables.last().unwrap().clone());
-        let table = SymbolTable::new(name.to_string());
+    fn enter_scope(&mut self, name: &str, typ: SymbolTableType, line_number: usize) {
+        let table = SymbolTable::new(name.to_string(), typ, line_number);
         self.tables.push(table);
     }
 
-    fn leave_block(&mut self) {
-        // Pop symbol table and add to sub table of parent table.
+    /// Pop symbol table and add to sub table of parent table.
+    fn leave_scope(&mut self) {
         let table = self.tables.pop().unwrap();
         self.tables.last_mut().unwrap().sub_tables.push(table);
     }
@@ -348,9 +373,9 @@ impl SymbolTableBuilder {
                 if let Some(expression) = returns {
                     self.scan_expression(expression, &ExpressionContext::Load)?;
                 }
-                self.enter_function(name, args)?;
+                self.enter_function(name, args, statement.location.row())?;
                 self.scan_statements(body)?;
-                self.leave_block();
+                self.leave_scope();
             }
             ClassDef {
                 name,
@@ -360,9 +385,9 @@ impl SymbolTableBuilder {
                 decorator_list,
             } => {
                 self.register_name(name, SymbolUsage::Assigned)?;
-                self.enter_block(name);
+                self.enter_scope(name, SymbolTableType::Class, statement.location.row());
                 self.scan_statements(body)?;
-                self.leave_block();
+                self.leave_scope();
                 self.scan_expressions(bases, &ExpressionContext::Load)?;
                 for keyword in keywords {
                     self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
@@ -612,9 +637,9 @@ impl SymbolTableBuilder {
                 }
             }
             Lambda { args, body } => {
-                self.enter_function("lambda", args)?;
+                self.enter_function("lambda", args, expression.location.row())?;
                 self.scan_expression(body, &ExpressionContext::Load)?;
-                self.leave_block();
+                self.leave_scope();
             }
             IfExpression { test, body, orelse } => {
                 self.scan_expression(test, &ExpressionContext::Load)?;
@@ -625,7 +650,12 @@ impl SymbolTableBuilder {
         Ok(())
     }
 
-    fn enter_function(&mut self, name: &str, args: &ast::Parameters) -> SymbolTableResult {
+    fn enter_function(
+        &mut self,
+        name: &str,
+        args: &ast::Parameters,
+        line_number: usize,
+    ) -> SymbolTableResult {
         // Evaluate eventual default parameters:
         self.scan_expressions(&args.defaults, &ExpressionContext::Load)?;
         for kw_default in &args.kw_defaults {
@@ -644,7 +674,7 @@ impl SymbolTableBuilder {
             self.scan_parameter_annotation(name)?;
         }
 
-        self.enter_block(name);
+        self.enter_scope(name, SymbolTableType::Function, line_number);
 
         // Fill scope with parameter names:
         self.scan_parameters(&args.args)?;

From a02ba6104fd16ac71f60a4ed36536e4b3f8f4955 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Fri, 30 Aug 2019 01:53:54 +0900
Subject: [PATCH 059/459] Fix scan_expression not to mark sequence as Load

---
 src/symboltable.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 3c70d4fc..df6c6d52 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -585,7 +585,7 @@ impl SymbolTableBuilder {
             }
             Bytes { .. } => {}
             Tuple { elements } | Set { elements } | List { elements } | Slice { elements } => {
-                self.scan_expressions(elements, &ExpressionContext::Load)?;
+                self.scan_expressions(elements, context)?;
             }
             Comprehension { kind, generators } => {
                 match **kind {

From efecef9d48b339dbc5d34eb8e5f39a02ea9032b0 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Fri, 30 Aug 2019 01:59:20 +0900
Subject: [PATCH 060/459] Remove duplicated contains_key test

---
 src/symboltable.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index df6c6d52..546e118a 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -710,7 +710,8 @@ impl SymbolTableBuilder {
         let location = Default::default();
 
         // Some checks:
-        if table.symbols.contains_key(name) {
+        let containing = table.symbols.contains_key(name);
+        if containing {
             // Role already set..
             match role {
                 SymbolUsage::Global => {
@@ -747,7 +748,7 @@ impl SymbolTableBuilder {
         }
 
         // Insert symbol when required:
-        if !table.symbols.contains_key(name) {
+        if !containing {
             let symbol = Symbol::new(name);
             table.symbols.insert(name.to_string(), symbol);
         }

From 9c17d982fff3458d6ff4c6d807bff84431383a54 Mon Sep 17 00:00:00 2001
From: Adam Kelly <adam@cthulahoops.org>
Date: Sat, 31 Aug 2019 16:39:27 +0100
Subject: [PATCH 061/459] Be extremely conservative about guessing scopes when
 we don't know.

---
 src/symboltable.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 3c70d4fc..ab931ac5 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -252,8 +252,8 @@ impl SymbolTableAnalyzer {
                         // Symbol is in some outer scope.
                         symbol.is_free = true;
                     } else {
-                        // Well, it must be a global then :)
-                        symbol.scope = SymbolScope::Global;
+                        // Don't make assumptions when we don't know.
+                        symbol.scope = SymbolScope::Unknown;
                     }
                 }
             }

From 024cf5db042c18053f4e2ed2b6a20b20445e0ba4 Mon Sep 17 00:00:00 2001
From: Windel Bouwman <windel@windel.nl>
Date: Sun, 1 Sep 2019 22:31:16 +0200
Subject: [PATCH 062/459] Add symboltable scope for comprehensions. Add _ast
 module nodes for comprehensions.

---
 src/compile.rs     |  4 ++++
 src/symboltable.rs | 31 ++++++++++++++++++++++++++++++-
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index 7cdfa1a3..7194c888 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1782,6 +1782,7 @@ impl<O: OutputStream> Compiler<O> {
             line_number,
             name.clone(),
         ));
+        self.enter_scope();
 
         // Create empty object of proper type:
         match kind {
@@ -1891,6 +1892,9 @@ impl<O: OutputStream> Compiler<O> {
         // Fetch code for listcomp function:
         let code = self.pop_code_object();
 
+        // Pop scope
+        self.leave_scope();
+
         // List comprehension code:
         self.emit(Instruction::LoadConst {
             value: bytecode::Constant::Code {
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 546e118a..cb367494 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -588,6 +588,23 @@ impl SymbolTableBuilder {
                 self.scan_expressions(elements, context)?;
             }
             Comprehension { kind, generators } => {
+                // Comprehensions are compiled as functions, so create a scope for them:
+                let scope_name = match **kind {
+                    ast::ComprehensionKind::GeneratorExpression { .. } => "genexpr",
+                    ast::ComprehensionKind::List { .. } => "listcomp",
+                    ast::ComprehensionKind::Set { .. } => "setcomp",
+                    ast::ComprehensionKind::Dict { .. } => "dictcomp",
+                };
+
+                self.enter_scope(
+                    scope_name,
+                    SymbolTableType::Function,
+                    expression.location.row(),
+                );
+
+                // Register the passed argument to the generator function as the name ".0"
+                self.register_name(".0", SymbolUsage::Parameter)?;
+
                 match **kind {
                     ast::ComprehensionKind::GeneratorExpression { ref element }
                     | ast::ComprehensionKind::List { ref element }
@@ -600,13 +617,25 @@ impl SymbolTableBuilder {
                     }
                 }
 
+                let mut is_first_generator = true;
                 for generator in generators {
                     self.scan_expression(&generator.target, &ExpressionContext::Store)?;
-                    self.scan_expression(&generator.iter, &ExpressionContext::Load)?;
+                    if is_first_generator {
+                        is_first_generator = false;
+                    } else {
+                        self.scan_expression(&generator.iter, &ExpressionContext::Load)?;
+                    }
+
                     for if_expr in &generator.ifs {
                         self.scan_expression(if_expr, &ExpressionContext::Load)?;
                     }
                 }
+
+                self.leave_scope();
+
+                // The first iterable is passed as an argument into the created function:
+                assert!(!generators.is_empty());
+                self.scan_expression(&generators[0].iter, &ExpressionContext::Load)?;
             }
             Call {
                 function,

From f3ecba02b121e1e9df089a19335080fd67893f38 Mon Sep 17 00:00:00 2001
From: Adam Kelly <adam@cthulahoops.org>
Date: Sat, 31 Aug 2019 18:15:56 +0100
Subject: [PATCH 063/459] Be somewhat less conservative about declaring things
 global.

---
 src/symboltable.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index ab931ac5..c07b5ae4 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -251,9 +251,12 @@ impl SymbolTableAnalyzer {
                     if found_in_outer_scope {
                         // Symbol is in some outer scope.
                         symbol.is_free = true;
-                    } else {
+                    } else if self.tables.is_empty() {
                         // Don't make assumptions when we don't know.
                         symbol.scope = SymbolScope::Unknown;
+                    } else {
+                        // If there are scopes above we can assume global.
+                        symbol.scope = SymbolScope::Global;
                     }
                 }
             }

From acce23b79b1c9f93c2d1ba1089a5f8bff1a9ecba Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Thu, 5 Sep 2019 00:27:16 -0500
Subject: [PATCH 064/459] Fix class vs method scopes

---
 src/compile.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index 7194c888..2324ac30 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -975,7 +975,7 @@ impl<O: OutputStream> Compiler<O> {
 
         // Turn code object into function object:
         self.emit(Instruction::MakeFunction {
-            flags: bytecode::FunctionOpArg::empty(),
+            flags: bytecode::FunctionOpArg::IS_CLASS,
         });
 
         self.emit(Instruction::LoadConst {

From d7f99dc7b1fd2d38c0c59585424bbd613b4160cb Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 7 Sep 2019 01:29:11 -0500
Subject: [PATCH 065/459] Fix class scopes by modifying symboltable

---
 src/compile.rs     |  2 +-
 src/symboltable.rs | 14 ++++++--------
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 2324ac30..7194c888 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -975,7 +975,7 @@ impl<O: OutputStream> Compiler<O> {
 
         // Turn code object into function object:
         self.emit(Instruction::MakeFunction {
-            flags: bytecode::FunctionOpArg::IS_CLASS,
+            flags: bytecode::FunctionOpArg::empty(),
         });
 
         self.emit(Instruction::LoadConst {
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 312650f9..43e0500e 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -61,7 +61,7 @@ impl SymbolTable {
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, PartialEq)]
 pub enum SymbolTableType {
     Module,
     Class,
@@ -241,12 +241,10 @@ impl SymbolTableAnalyzer {
                 } else {
                     // Interesting stuff about the __class__ variable:
                     // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
-                    let found_in_outer_scope = (symbol.name == "__class__")
-                        || self
-                            .tables
-                            .iter()
-                            .skip(1)
-                            .any(|t| t.symbols.contains_key(&symbol.name));
+                    let found_in_outer_scope = symbol.name == "__class__"
+                        || self.tables.iter().skip(1).any(|t| {
+                            t.typ != SymbolTableType::Class && t.symbols.contains_key(&symbol.name)
+                        });
 
                     if found_in_outer_scope {
                         // Symbol is in some outer scope.
@@ -387,7 +385,6 @@ impl SymbolTableBuilder {
                 keywords,
                 decorator_list,
             } => {
-                self.register_name(name, SymbolUsage::Assigned)?;
                 self.enter_scope(name, SymbolTableType::Class, statement.location.row());
                 self.scan_statements(body)?;
                 self.leave_scope();
@@ -396,6 +393,7 @@ impl SymbolTableBuilder {
                     self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
                 }
                 self.scan_expressions(decorator_list, &ExpressionContext::Load)?;
+                self.register_name(name, SymbolUsage::Assigned)?;
             }
             Expression { expression } => {
                 self.scan_expression(expression, &ExpressionContext::Load)?

From 5919a1e1ba1b8b2cca0060e3da3d2ee50bebf407 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 7 Sep 2019 18:11:47 -0500
Subject: [PATCH 066/459] Don't clone SymbolTables when analyzing

---
 src/symboltable.rs | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 43e0500e..1578f479 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -61,7 +61,7 @@ impl SymbolTable {
     }
 }
 
-#[derive(Clone, PartialEq)]
+#[derive(Clone, Copy, PartialEq)]
 pub enum SymbolTableType {
     Module,
     Class,
@@ -179,24 +179,24 @@ fn analyze_symbol_table(symbol_table: &mut SymbolTable) -> SymbolTableResult {
 /// build symbol table structure. It will mark variables
 /// as local variables for example.
 #[derive(Default)]
-struct SymbolTableAnalyzer {
-    tables: Vec<SymbolTable>,
+struct SymbolTableAnalyzer<'a> {
+    tables: Vec<(&'a mut IndexMap<String, Symbol>, SymbolTableType)>,
 }
 
-impl SymbolTableAnalyzer {
-    fn analyze_symbol_table(&mut self, symbol_table: &mut SymbolTable) -> SymbolTableResult {
-        // Store a copy to determine the parent.
-        // TODO: this should be improved to resolve this clone action.
-        self.tables.push(symbol_table.clone());
+impl<'a> SymbolTableAnalyzer<'a> {
+    fn analyze_symbol_table(&mut self, symbol_table: &'a mut SymbolTable) -> SymbolTableResult {
+        let symbols = &mut symbol_table.symbols;
+        let sub_tables = &mut symbol_table.sub_tables;
 
+        self.tables.push((symbols, symbol_table.typ));
         // Analyze sub scopes:
-        for sub_table in &mut symbol_table.sub_tables {
+        for sub_table in sub_tables {
             self.analyze_symbol_table(sub_table)?;
         }
-        self.tables.pop();
+        let (symbols, _) = self.tables.pop().unwrap();
 
         // Analyze symbols:
-        for symbol in symbol_table.symbols.values_mut() {
+        for symbol in symbols.values_mut() {
             self.analyze_symbol(symbol)?;
         }
 
@@ -207,11 +207,11 @@ impl SymbolTableAnalyzer {
         match symbol.scope {
             SymbolScope::Nonlocal => {
                 // check if name is defined in parent table!
-                let parent_symbol_table: Option<&SymbolTable> = self.tables.last();
+                let parent_symbol_table = self.tables.last();
                 // symbol.table.borrow().parent.clone();
 
-                if let Some(table) = parent_symbol_table {
-                    if !table.symbols.contains_key(&symbol.name) {
+                if let Some((symbols, _)) = parent_symbol_table {
+                    if !symbols.contains_key(&symbol.name) {
                         return Err(SymbolTableError {
                             error: format!("no binding for nonlocal '{}' found", symbol.name),
                             location: Default::default(),
@@ -242,8 +242,8 @@ impl SymbolTableAnalyzer {
                     // Interesting stuff about the __class__ variable:
                     // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
                     let found_in_outer_scope = symbol.name == "__class__"
-                        || self.tables.iter().skip(1).any(|t| {
-                            t.typ != SymbolTableType::Class && t.symbols.contains_key(&symbol.name)
+                        || self.tables.iter().skip(1).any(|(symbols, typ)| {
+                            *typ != SymbolTableType::Class && symbols.contains_key(&symbol.name)
                         });
 
                     if found_in_outer_scope {

From c852c5d2721fd017fb2a61528618f3b8e060f6d5 Mon Sep 17 00:00:00 2001
From: HyeockJinKim <kherootz@gmail.com>
Date: Thu, 12 Sep 2019 01:01:04 +0900
Subject: [PATCH 067/459] nonlocal generate syntax error

If the scope depth is less than 2,
a syntax error occurs in nonlocal.

Fixes #1360
---
 src/symboltable.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 1578f479..f1cc1a8b 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -211,7 +211,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
                 // symbol.table.borrow().parent.clone();
 
                 if let Some((symbols, _)) = parent_symbol_table {
-                    if !symbols.contains_key(&symbol.name) {
+                    let scope_depth = self.tables.len();
+                    if !symbols.contains_key(&symbol.name) || scope_depth < 2 {
                         return Err(SymbolTableError {
                             error: format!("no binding for nonlocal '{}' found", symbol.name),
                             location: Default::default(),

From 5ce3f2c0566a27152dcc6bea7203aeaf3d8fd053 Mon Sep 17 00:00:00 2001
From: j30ng <leej0116@outlook.com>
Date: Thu, 12 Sep 2019 03:15:03 +0900
Subject: [PATCH 068/459] Return SyntaxError on Invalid Star Expression

---
 src/compile.rs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 7194c888..b42496ad 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1616,10 +1616,12 @@ impl<O: OutputStream> Compiler<O> {
             Comprehension { kind, generators } => {
                 self.compile_comprehension(kind, generators)?;
             }
-            Starred { value } => {
-                self.compile_expression(value)?;
-                self.emit(Instruction::Unpack);
-                panic!("We should not just unpack a starred args, since the size is unknown.");
+            Starred { .. } => {
+                use std::string::String;
+                return Err(CompileError {
+                    error: CompileErrorType::SyntaxError(String::from("Invalid starred expression")),
+                    location: self.current_source_location.clone(),
+                });
             }
             IfExpression { test, body, orelse } => {
                 let no_label = self.new_label();

From 3b99c07a1efcca480a3aa1860ab45aa4d122d7db Mon Sep 17 00:00:00 2001
From: j30ng <leej0116@outlook.com>
Date: Thu, 12 Sep 2019 04:36:31 +0900
Subject: [PATCH 069/459] Format Code with rustfmt

---
 src/compile.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index b42496ad..cf6405d8 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1619,7 +1619,9 @@ impl<O: OutputStream> Compiler<O> {
             Starred { .. } => {
                 use std::string::String;
                 return Err(CompileError {
-                    error: CompileErrorType::SyntaxError(String::from("Invalid starred expression")),
+                    error: CompileErrorType::SyntaxError(String::from(
+                        "Invalid starred expression",
+                    )),
                     location: self.current_source_location.clone(),
                 });
             }

From df4be1f9ced7c72ed7c6c187d3b8b98637bec1f3 Mon Sep 17 00:00:00 2001
From: j30ng <leej0116@outlook.com>
Date: Thu, 12 Sep 2019 15:55:36 +0900
Subject: [PATCH 070/459] Remove Enum Instruction::Unpack

---
 src/compile.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index cf6405d8..890ff8e3 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1617,9 +1617,8 @@ impl<O: OutputStream> Compiler<O> {
                 self.compile_comprehension(kind, generators)?;
             }
             Starred { .. } => {
-                use std::string::String;
                 return Err(CompileError {
-                    error: CompileErrorType::SyntaxError(String::from(
+                    error: CompileErrorType::SyntaxError(std::string::String::from(
                         "Invalid starred expression",
                     )),
                     location: self.current_source_location.clone(),

From e5f6d11a12c892e1fe7127e356fce5b71e57a51d Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 16 Aug 2019 20:15:10 -0500
Subject: [PATCH 071/459] Add optimization to remove unused LoadConst
 operations

---
 src/peephole.rs               | 112 +++++-----------------------------
 src/peephole/optimizations.rs | 110 +++++++++++++++++++++++++++++++++
 2 files changed, 125 insertions(+), 97 deletions(-)
 create mode 100644 src/peephole/optimizations.rs

diff --git a/src/peephole.rs b/src/peephole.rs
index c67a16a8..2eff20c7 100644
--- a/src/peephole.rs
+++ b/src/peephole.rs
@@ -1,6 +1,8 @@
 use crate::output_stream::OutputStream;
 use arrayvec::ArrayVec;
-use rustpython_bytecode::bytecode::{self, CodeObject, Instruction, Label, Location};
+use rustpython_bytecode::bytecode::{CodeObject, Instruction, Label, Location};
+
+pub mod optimizations;
 
 const PEEPHOLE_BUFFER_SIZE: usize = 20;
 
@@ -47,6 +49,13 @@ impl<O: OutputStream> From<PeepholeOptimizer<O>> for CodeObject {
     }
 }
 
+#[macro_export]
+macro_rules! apply_optimizations {
+    ($buf:expr, $($opt:ident),*$(,)?) => {{
+        $($crate::peephole::optimizations::$opt($buf);)*
+    }};
+}
+
 impl<O: OutputStream> PeepholeOptimizer<O> {
     pub fn new(inner: O) -> Self {
         PeepholeOptimizer {
@@ -81,6 +90,10 @@ impl<O: OutputStream> PeepholeOptimizer<O> {
             Self::inner_emit(&mut self.inner, instruction, meta);
         }
     }
+
+    fn optimize(&mut self) {
+        apply_optimizations!(self, operator, unpack, useless_const);
+    }
 }
 
 impl<O> OutputStream for PeepholeOptimizer<O>
@@ -89,7 +102,7 @@ where
 {
     fn emit(&mut self, instruction: Instruction, loc: Location) {
         self.push(instruction, loc.into());
-        optimize(self);
+        self.optimize();
     }
     fn set_label(&mut self, label: Label) {
         if let Some(instr) = self.buffer.last_mut() {
@@ -116,98 +129,3 @@ pub trait OptimizationBuffer {
     fn emit(&mut self, instruction: Instruction, meta: InstructionMetadata);
     fn pop(&mut self) -> (Instruction, InstructionMetadata);
 }
-
-macro_rules! lc {
-    ($name:ident {$($field:tt)*}) => {
-        Instruction::LoadConst {
-            value: bytecode::Constant::$name {$($field)*},
-        }
-    };
-    ($name:ident, $($value:tt)*) => {
-        lc!($name { value: $($value)* })
-    };
-}
-macro_rules! emitconst {
-    ($buf:expr, [$($metas:expr),*], $($arg:tt)*) => {
-        $buf.emit(
-            lc!($($arg)*),
-            InstructionMetadata::from(vec![$($metas),*]),
-        )
-    };
-}
-
-pub fn optimize(buf: &mut impl OptimizationBuffer) {
-    optimize_operator(buf);
-    optimize_unpack(buf);
-}
-
-fn optimize_operator(buf: &mut impl OptimizationBuffer) {
-    let (instruction, meta) = buf.pop();
-    if let Instruction::BinaryOperation { op, inplace } = instruction {
-        let (rhs, rhs_meta) = buf.pop();
-        let (lhs, lhs_meta) = buf.pop();
-        macro_rules! op {
-            ($op:ident) => {
-                bytecode::BinaryOperator::$op
-            };
-        }
-        match (op, lhs, rhs) {
-            (op!(Add), lc!(Integer, lhs), lc!(Integer, rhs)) => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Integer, lhs + rhs)
-            }
-            (op!(Subtract), lc!(Integer, lhs), lc!(Integer, rhs)) => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Integer, lhs - rhs)
-            }
-            (op!(Add), lc!(Float, lhs), lc!(Float, rhs)) => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs + rhs)
-            }
-            (op!(Subtract), lc!(Float, lhs), lc!(Float, rhs)) => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs - rhs)
-            }
-            (op!(Multiply), lc!(Float, lhs), lc!(Float, rhs)) => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs * rhs)
-            }
-            (op!(Divide), lc!(Float, lhs), lc!(Float, rhs)) => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs / rhs)
-            }
-            (op!(Power), lc!(Float, lhs), lc!(Float, rhs)) => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs.powf(rhs))
-            }
-            (op!(Add), lc!(String, mut lhs), lc!(String, rhs)) => {
-                lhs.push_str(&rhs);
-                emitconst!(buf, [lhs_meta, rhs_meta], String, lhs);
-            }
-            (op, lhs, rhs) => {
-                buf.emit(lhs, lhs_meta);
-                buf.emit(rhs, rhs_meta);
-                buf.emit(Instruction::BinaryOperation { op, inplace }, meta);
-            }
-        }
-    } else {
-        buf.emit(instruction, meta)
-    }
-}
-
-fn optimize_unpack(buf: &mut impl OptimizationBuffer) {
-    let (instruction, meta) = buf.pop();
-    if let Instruction::UnpackSequence { size } = instruction {
-        let (arg, arg_meta) = buf.pop();
-        match arg {
-            Instruction::BuildTuple {
-                size: tup_size,
-                unpack,
-            } if !unpack && tup_size == size => {
-                buf.emit(
-                    Instruction::Reverse { amount: size },
-                    vec![arg_meta, meta].into(),
-                );
-            }
-            arg => {
-                buf.emit(arg, arg_meta);
-                buf.emit(instruction, meta);
-            }
-        }
-    } else {
-        buf.emit(instruction, meta)
-    }
-}
diff --git a/src/peephole/optimizations.rs b/src/peephole/optimizations.rs
new file mode 100644
index 00000000..ee8f1f0d
--- /dev/null
+++ b/src/peephole/optimizations.rs
@@ -0,0 +1,110 @@
+use rustpython_bytecode::bytecode::{self, Instruction};
+
+use super::{InstructionMetadata, OptimizationBuffer};
+
+macro_rules! lc {
+    ($name:ident {$($field:tt)*}) => {
+        Instruction::LoadConst {
+            value: bytecode::Constant::$name {$($field)*},
+        }
+    };
+    ($name:ident, $($value:tt)*) => {
+        lc!($name { value: $($value)* })
+    };
+}
+macro_rules! emitconst {
+    ($buf:expr, [$($metas:expr),*], $($arg:tt)*) => {
+        $buf.emit(
+            lc!($($arg)*),
+            InstructionMetadata::from(vec![$($metas),*]),
+        )
+    };
+}
+
+pub fn operator(buf: &mut impl OptimizationBuffer) {
+    let (instruction, meta) = buf.pop();
+    if let Instruction::BinaryOperation { op, inplace } = instruction {
+        let (rhs, rhs_meta) = buf.pop();
+        let (lhs, lhs_meta) = buf.pop();
+        macro_rules! op {
+            ($op:ident) => {
+                bytecode::BinaryOperator::$op
+            };
+        }
+        match (op, lhs, rhs) {
+            (op!(Add), lc!(Integer, lhs), lc!(Integer, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Integer, lhs + rhs)
+            }
+            (op!(Subtract), lc!(Integer, lhs), lc!(Integer, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Integer, lhs - rhs)
+            }
+            (op!(Add), lc!(Float, lhs), lc!(Float, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs + rhs)
+            }
+            (op!(Subtract), lc!(Float, lhs), lc!(Float, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs - rhs)
+            }
+            (op!(Multiply), lc!(Float, lhs), lc!(Float, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs * rhs)
+            }
+            (op!(Divide), lc!(Float, lhs), lc!(Float, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs / rhs)
+            }
+            (op!(Power), lc!(Float, lhs), lc!(Float, rhs)) => {
+                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs.powf(rhs))
+            }
+            (op!(Add), lc!(String, mut lhs), lc!(String, rhs)) => {
+                lhs.push_str(&rhs);
+                emitconst!(buf, [lhs_meta, rhs_meta], String, lhs);
+            }
+            (op, lhs, rhs) => {
+                buf.emit(lhs, lhs_meta);
+                buf.emit(rhs, rhs_meta);
+                buf.emit(Instruction::BinaryOperation { op, inplace }, meta);
+            }
+        }
+    } else {
+        buf.emit(instruction, meta)
+    }
+}
+
+pub fn unpack(buf: &mut impl OptimizationBuffer) {
+    let (instruction, meta) = buf.pop();
+    if let Instruction::UnpackSequence { size } = instruction {
+        let (arg, arg_meta) = buf.pop();
+        match arg {
+            Instruction::BuildTuple {
+                size: tup_size,
+                unpack,
+            } if !unpack && tup_size == size => {
+                buf.emit(
+                    Instruction::Reverse { amount: size },
+                    vec![arg_meta, meta].into(),
+                );
+            }
+            arg => {
+                buf.emit(arg, arg_meta);
+                buf.emit(instruction, meta);
+            }
+        }
+    } else {
+        buf.emit(instruction, meta)
+    }
+}
+
+pub fn useless_const(buf: &mut impl OptimizationBuffer) {
+    let (instruction, meta) = buf.pop();
+    if instruction == Instruction::Pop {
+        let (arg, arg_meta) = buf.pop();
+        if let Instruction::LoadConst { .. } = arg {
+            // just ignore it all
+            drop(arg);
+            drop(instruction);
+        } else {
+            buf.emit(arg, arg_meta);
+            buf.emit(instruction, meta);
+        }
+    } else {
+        buf.emit(instruction, meta);
+    }
+}

From b8726f03dfb04eddcac2ec0e02dd7b0d6e07e9b6 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 14 Sep 2019 15:16:02 -0500
Subject: [PATCH 072/459] Don't emit None, ReturnValue if the last statement in
 a function is a return

---
 src/compile.rs | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 890ff8e3..b1b5f62b 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -842,15 +842,21 @@ impl<O: OutputStream> Compiler<O> {
 
         let mut flags = self.enter_function(name, args)?;
 
-        let (new_body, doc_str) = get_doc(body);
+        let (body, doc_str) = get_doc(body);
 
-        self.compile_statements(new_body)?;
+        self.compile_statements(body)?;
 
         // Emit None at end:
-        self.emit(Instruction::LoadConst {
-            value: bytecode::Constant::None,
-        });
-        self.emit(Instruction::ReturnValue);
+        match body.last().map(|s| &s.node) {
+            Some(ast::StatementType::Return { .. }) => {}
+            _ => {
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::None,
+                });
+                self.emit(Instruction::ReturnValue);
+            }
+        }
+
         let code = self.pop_code_object();
         self.leave_scope();
 

From bff58fd6268d011ad5b50626d9f8f658e4c97eb4 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sun, 15 Sep 2019 18:34:48 -0500
Subject: [PATCH 073/459] Remove useless_const optimization

There could be jump depending on the Pop being there, and then it
breaks.
---
 src/compile.rs                |  4 +++-
 src/peephole.rs               |  4 +---
 src/peephole/optimizations.rs | 26 +++++++-------------------
 3 files changed, 11 insertions(+), 23 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index b1b5f62b..403b1487 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -848,7 +848,9 @@ impl<O: OutputStream> Compiler<O> {
 
         // Emit None at end:
         match body.last().map(|s| &s.node) {
-            Some(ast::StatementType::Return { .. }) => {}
+            Some(ast::StatementType::Return { .. }) => {
+                // the last instruction is a ReturnValue already, we don't need to emit it
+            }
             _ => {
                 self.emit(Instruction::LoadConst {
                     value: bytecode::Constant::None,
diff --git a/src/peephole.rs b/src/peephole.rs
index 2eff20c7..3a64380b 100644
--- a/src/peephole.rs
+++ b/src/peephole.rs
@@ -92,7 +92,7 @@ impl<O: OutputStream> PeepholeOptimizer<O> {
     }
 
     fn optimize(&mut self) {
-        apply_optimizations!(self, operator, unpack, useless_const);
+        apply_optimizations!(self, operator, unpack);
     }
 }
 
@@ -123,8 +123,6 @@ impl<O: OutputStream> OptimizationBuffer for PeepholeOptimizer<O> {
     }
 }
 
-// OPTIMIZATION
-
 pub trait OptimizationBuffer {
     fn emit(&mut self, instruction: Instruction, meta: InstructionMetadata);
     fn pop(&mut self) -> (Instruction, InstructionMetadata);
diff --git a/src/peephole/optimizations.rs b/src/peephole/optimizations.rs
index ee8f1f0d..b0fb7531 100644
--- a/src/peephole/optimizations.rs
+++ b/src/peephole/optimizations.rs
@@ -2,6 +2,11 @@ use rustpython_bytecode::bytecode::{self, Instruction};
 
 use super::{InstructionMetadata, OptimizationBuffer};
 
+macro_rules! metas {
+    [$($metas:expr),*$(,)?] => {
+        InstructionMetadata::from(vec![$($metas),*])
+    };
+}
 macro_rules! lc {
     ($name:ident {$($field:tt)*}) => {
         Instruction::LoadConst {
@@ -13,10 +18,10 @@ macro_rules! lc {
     };
 }
 macro_rules! emitconst {
-    ($buf:expr, [$($metas:expr),*], $($arg:tt)*) => {
+    ($buf:expr, [$($metas:expr),*$(,)?], $($arg:tt)*) => {
         $buf.emit(
             lc!($($arg)*),
-            InstructionMetadata::from(vec![$($metas),*]),
+            metas![$($metas),*],
         )
     };
 }
@@ -91,20 +96,3 @@ pub fn unpack(buf: &mut impl OptimizationBuffer) {
         buf.emit(instruction, meta)
     }
 }
-
-pub fn useless_const(buf: &mut impl OptimizationBuffer) {
-    let (instruction, meta) = buf.pop();
-    if instruction == Instruction::Pop {
-        let (arg, arg_meta) = buf.pop();
-        if let Instruction::LoadConst { .. } = arg {
-            // just ignore it all
-            drop(arg);
-            drop(instruction);
-        } else {
-            buf.emit(arg, arg_meta);
-            buf.emit(instruction, meta);
-        }
-    } else {
-        buf.emit(instruction, meta);
-    }
-}

From 18c243854e428ce32fcf17399bd47455cb430d79 Mon Sep 17 00:00:00 2001
From: HyeockJinKim <kherootz@gmail.com>
Date: Wed, 18 Sep 2019 01:33:31 +0900
Subject: [PATCH 074/459] Fix global declaration error in the function

Fix Global symbol to prevent syntax error
when global declaration for same variable in function.

Fixes #1353
---
 src/symboltable.rs | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index f1cc1a8b..d64dd5b0 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -746,10 +746,15 @@ impl SymbolTableBuilder {
             // Role already set..
             match role {
                 SymbolUsage::Global => {
-                    return Err(SymbolTableError {
-                        error: format!("name '{}' is used prior to global declaration", name),
-                        location,
-                    })
+                    let symbol = table.symbols.get(name).unwrap();
+                    if let SymbolScope::Global = symbol.scope {
+                        // Ok
+                    } else {
+                        return Err(SymbolTableError {
+                            error: format!("name '{}' is used prior to global declaration", name),
+                            location,
+                        });
+                    }
                 }
                 SymbolUsage::Nonlocal => {
                     return Err(SymbolTableError {
@@ -806,6 +811,8 @@ impl SymbolTableBuilder {
             SymbolUsage::Global => {
                 if let SymbolScope::Unknown = symbol.scope {
                     symbol.scope = SymbolScope::Global;
+                } else if let SymbolScope::Global = symbol.scope {
+                    // Global scope can be set to global
                 } else {
                     return Err(SymbolTableError {
                         error: format!("Symbol {} scope cannot be set to global, since its scope was already determined otherwise.", name),

From a29fb0de1287293d1200d7db4ba7520922b00d20 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 25 Sep 2019 11:37:02 -0500
Subject: [PATCH 075/459] Bump crate versions

---
 Cargo.toml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index ef550980..28dc371c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "rustpython-compiler"
-version = "0.1.0"
+version = "0.1.1"
 description = "Compiler for python code into bytecode for the rustpython VM."
 authors = ["RustPython Team"]
 repository = "https://github.com/RustPython/RustPython"
@@ -9,8 +9,8 @@ edition = "2018"
 
 [dependencies]
 indexmap = "1.0"
-rustpython-bytecode = { path = "../bytecode", version = "0.1.0" }
-rustpython-parser = { path = "../parser", version = "0.1.0" }
+rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
+rustpython-parser = { path = "../parser", version = "0.1.1" }
 num-complex = { version = "0.2", features = ["serde"] }
 log = "0.3"
 arrayvec = "0.4"

From fee1b6f2c7aa0cc099d0a7709c016a92719978d3 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Tue, 10 Sep 2019 11:37:11 -0500
Subject: [PATCH 076/459] Only overwrite the locals in __build_class__ if it's
 a class

---
 src/compile.rs     | 13 +++++++++++--
 src/symboltable.rs |  2 ++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 403b1487..838afe31 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -955,12 +955,21 @@ impl<O: OutputStream> Compiler<O> {
 
         self.emit(Instruction::LoadName {
             name: "__name__".to_string(),
-            scope: bytecode::NameScope::Free,
+            scope: bytecode::NameScope::Global,
         });
         self.emit(Instruction::StoreName {
             name: "__module__".to_string(),
             scope: bytecode::NameScope::Free,
         });
+        self.emit(Instruction::LoadConst {
+            value: bytecode::Constant::String {
+                value: qualified_name.clone(),
+            },
+        });
+        self.emit(Instruction::StoreName {
+            name: "__qualname__".to_string(),
+            scope: bytecode::NameScope::Free,
+        });
         self.compile_statements(new_body)?;
         self.emit(Instruction::LoadConst {
             value: bytecode::Constant::None,
@@ -983,7 +992,7 @@ impl<O: OutputStream> Compiler<O> {
 
         // Turn code object into function object:
         self.emit(Instruction::MakeFunction {
-            flags: bytecode::FunctionOpArg::empty(),
+            flags: bytecode::FunctionOpArg::NO_NEW_LOCALS,
         });
 
         self.emit(Instruction::LoadConst {
diff --git a/src/symboltable.rs b/src/symboltable.rs
index d64dd5b0..61a76ddf 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -387,6 +387,8 @@ impl SymbolTableBuilder {
                 decorator_list,
             } => {
                 self.enter_scope(name, SymbolTableType::Class, statement.location.row());
+                self.register_name("__module__", SymbolUsage::Assigned)?;
+                self.register_name("__qualname__", SymbolUsage::Assigned)?;
                 self.scan_statements(body)?;
                 self.leave_scope();
                 self.scan_expressions(bases, &ExpressionContext::Load)?;

From 0c3482e03abef22268b731f751b78267c372bbae Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 28 Sep 2019 00:41:37 -0500
Subject: [PATCH 077/459] Invert the inverted boolean

---
 src/compile.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 838afe31..3cdcd187 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -672,7 +672,7 @@ impl<O: OutputStream> Compiler<O> {
         ));
         self.enter_scope();
 
-        let mut flags = bytecode::FunctionOpArg::empty();
+        let mut flags = bytecode::FunctionOpArg::default();
         if have_defaults {
             flags |= bytecode::FunctionOpArg::HAS_DEFAULTS;
         }
@@ -992,7 +992,7 @@ impl<O: OutputStream> Compiler<O> {
 
         // Turn code object into function object:
         self.emit(Instruction::MakeFunction {
-            flags: bytecode::FunctionOpArg::NO_NEW_LOCALS,
+            flags: bytecode::FunctionOpArg::default() & !bytecode::FunctionOpArg::NEW_LOCALS,
         });
 
         self.emit(Instruction::LoadConst {
@@ -1929,7 +1929,7 @@ impl<O: OutputStream> Compiler<O> {
 
         // Turn code object into function object:
         self.emit(Instruction::MakeFunction {
-            flags: bytecode::FunctionOpArg::empty(),
+            flags: bytecode::FunctionOpArg::default(),
         });
 
         // Evaluate iterated item:

From 6c88f6af3d2af653f45635472b173594740ce9b3 Mon Sep 17 00:00:00 2001
From: Seo Sanghyeon <sanxiyn@gmail.com>
Date: Thu, 3 Oct 2019 00:03:08 +0900
Subject: [PATCH 078/459] Optimize BuildMap bytecode emission

---
 Cargo.toml     |   1 +
 src/compile.rs | 124 +++++++++++++++++++++++++++++++------------------
 2 files changed, 79 insertions(+), 46 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 28dc371c..32abc885 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,6 +9,7 @@ edition = "2018"
 
 [dependencies]
 indexmap = "1.0"
+itertools = "0.8.0"
 rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
 rustpython-parser = { path = "../parser", version = "0.1.1" }
 num-complex = { version = "0.2", features = ["serde"] }
diff --git a/src/compile.rs b/src/compile.rs
index 3cdcd187..a7e81460 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -11,6 +11,7 @@ use crate::peephole::PeepholeOptimizer;
 use crate::symboltable::{
     make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope, SymbolTable,
 };
+use itertools::Itertools;
 use num_complex::Complex64;
 use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Label, Varargs};
 use rustpython_parser::{ast, parser};
@@ -1438,6 +1439,47 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
+    fn compile_dict(
+        &mut self,
+        pairs: &[(Option<ast::Expression>, ast::Expression)],
+    ) -> Result<(), CompileError> {
+        let mut size = 0;
+        let mut has_unpacking = false;
+        for (is_unpacking, subpairs) in &pairs.iter().group_by(|e| e.0.is_none()) {
+            if is_unpacking {
+                for (_, value) in subpairs {
+                    self.compile_expression(value)?;
+                    size += 1;
+                }
+                has_unpacking = true;
+            } else {
+                let mut subsize = 0;
+                for (key, value) in subpairs {
+                    if let Some(key) = key {
+                        self.compile_expression(key)?;
+                        self.compile_expression(value)?;
+                        subsize += 1;
+                    }
+                }
+                self.emit(Instruction::BuildMap {
+                    size: subsize,
+                    unpack: false,
+                });
+                size += 1;
+            }
+        }
+        if size == 0 {
+            self.emit(Instruction::BuildMap {
+                size,
+                unpack: false,
+            });
+        }
+        if size > 1 || has_unpacking {
+            self.emit(Instruction::BuildMap { size, unpack: true });
+        }
+        Ok(())
+    }
+
     fn compile_expression(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
         trace!("Compiling {:?}", expression);
         self.set_source_location(&expression.location);
@@ -1521,27 +1563,7 @@ impl<O: OutputStream> Compiler<O> {
                 });
             }
             Dict { elements } => {
-                let size = elements.len();
-                let has_double_star = elements.iter().any(|e| e.0.is_none());
-                for (key, value) in elements {
-                    if let Some(key) = key {
-                        self.compile_expression(key)?;
-                        self.compile_expression(value)?;
-                        if has_double_star {
-                            self.emit(Instruction::BuildMap {
-                                size: 1,
-                                unpack: false,
-                            });
-                        }
-                    } else {
-                        // dict unpacking
-                        self.compile_expression(value)?;
-                    }
-                }
-                self.emit(Instruction::BuildMap {
-                    size,
-                    unpack: has_double_star,
-                });
+                self.compile_dict(elements)?;
             }
             Slice { elements } => {
                 let size = elements.len();
@@ -1658,6 +1680,40 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
+    fn compile_keywords(&mut self, keywords: &[ast::Keyword]) -> Result<(), CompileError> {
+        let mut size = 0;
+        for (is_unpacking, subkeywords) in &keywords.iter().group_by(|e| e.name.is_none()) {
+            if is_unpacking {
+                for keyword in subkeywords {
+                    self.compile_expression(&keyword.value)?;
+                    size += 1;
+                }
+            } else {
+                let mut subsize = 0;
+                for keyword in subkeywords {
+                    if let Some(name) = &keyword.name {
+                        self.emit(Instruction::LoadConst {
+                            value: bytecode::Constant::String {
+                                value: name.to_string(),
+                            },
+                        });
+                        self.compile_expression(&keyword.value)?;
+                        subsize += 1;
+                    }
+                }
+                self.emit(Instruction::BuildMap {
+                    size: subsize,
+                    unpack: false,
+                });
+                size += 1;
+            }
+        }
+        if size > 1 {
+            self.emit(Instruction::BuildMap { size, unpack: true });
+        }
+        Ok(())
+    }
+
     fn compile_call(
         &mut self,
         function: &ast::Expression,
@@ -1680,31 +1736,7 @@ impl<O: OutputStream> Compiler<O> {
 
             // Create an optional map with kw-args:
             if !keywords.is_empty() {
-                for keyword in keywords {
-                    if let Some(name) = &keyword.name {
-                        self.emit(Instruction::LoadConst {
-                            value: bytecode::Constant::String {
-                                value: name.to_string(),
-                            },
-                        });
-                        self.compile_expression(&keyword.value)?;
-                        if has_double_star {
-                            self.emit(Instruction::BuildMap {
-                                size: 1,
-                                unpack: false,
-                            });
-                        }
-                    } else {
-                        // This means **kwargs!
-                        self.compile_expression(&keyword.value)?;
-                    }
-                }
-
-                self.emit(Instruction::BuildMap {
-                    size: keywords.len(),
-                    unpack: has_double_star,
-                });
-
+                self.compile_keywords(keywords)?;
                 self.emit(Instruction::CallFunction {
                     typ: CallType::Ex(true),
                 });

From fe4f8237ada318c27c39380e78a8d7f0d15f3459 Mon Sep 17 00:00:00 2001
From: Seo Sanghyeon <sanxiyn@gmail.com>
Date: Fri, 4 Oct 2019 19:29:57 +0900
Subject: [PATCH 079/459] Raise TypeError for duplicate keyword arguments

---
 src/compile.rs | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index a7e81460..c457ae8e 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -658,6 +658,7 @@ impl<O: OutputStream> Compiler<O> {
             self.emit(Instruction::BuildMap {
                 size: num_kw_only_defaults,
                 unpack: false,
+                for_call: false,
             });
         }
 
@@ -896,6 +897,7 @@ impl<O: OutputStream> Compiler<O> {
             self.emit(Instruction::BuildMap {
                 size: num_annotations,
                 unpack: false,
+                for_call: false,
             });
         }
 
@@ -1464,6 +1466,7 @@ impl<O: OutputStream> Compiler<O> {
                 self.emit(Instruction::BuildMap {
                     size: subsize,
                     unpack: false,
+                    for_call: false,
                 });
                 size += 1;
             }
@@ -1472,10 +1475,15 @@ impl<O: OutputStream> Compiler<O> {
             self.emit(Instruction::BuildMap {
                 size,
                 unpack: false,
+                for_call: false,
             });
         }
         if size > 1 || has_unpacking {
-            self.emit(Instruction::BuildMap { size, unpack: true });
+            self.emit(Instruction::BuildMap {
+                size,
+                unpack: true,
+                for_call: false,
+            });
         }
         Ok(())
     }
@@ -1704,12 +1712,17 @@ impl<O: OutputStream> Compiler<O> {
                 self.emit(Instruction::BuildMap {
                     size: subsize,
                     unpack: false,
+                    for_call: false,
                 });
                 size += 1;
             }
         }
         if size > 1 {
-            self.emit(Instruction::BuildMap { size, unpack: true });
+            self.emit(Instruction::BuildMap {
+                size,
+                unpack: true,
+                for_call: true,
+            });
         }
         Ok(())
     }
@@ -1855,6 +1868,7 @@ impl<O: OutputStream> Compiler<O> {
                 self.emit(Instruction::BuildMap {
                     size: 0,
                     unpack: false,
+                    for_call: false,
                 });
             }
         }

From a1e228d4eed357e882b2508d2dc13806a74c16ff Mon Sep 17 00:00:00 2001
From: Seo Sanghyeon <sanxiyn@gmail.com>
Date: Sat, 5 Oct 2019 09:19:33 +0900
Subject: [PATCH 080/459] Implement TabError

---
 src/error.rs | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/error.rs b/src/error.rs
index 679a914b..62546ddb 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -1,4 +1,4 @@
-use rustpython_parser::error::{ParseError, ParseErrorType};
+use rustpython_parser::error::{LexicalErrorType, ParseError, ParseErrorType};
 use rustpython_parser::location::Location;
 
 use std::error::Error;
@@ -40,6 +40,19 @@ pub enum CompileErrorType {
     InvalidYield,
 }
 
+impl CompileError {
+    pub fn is_tab_error(&self) -> bool {
+        if let CompileErrorType::Parse(parse) = &self.error {
+            if let ParseErrorType::Lexical(lex) = parse {
+                if let LexicalErrorType::TabError = lex {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+}
+
 impl fmt::Display for CompileError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match &self.error {

From 2326b114c21cbbaf6f9538cf304b2a63660fce00 Mon Sep 17 00:00:00 2001
From: "Chris West (Faux)" <git@goeswhere.com>
Date: Thu, 10 Oct 2019 21:07:26 +0100
Subject: [PATCH 081/459] upgrade low-risk deps

---
 Cargo.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 32abc885..54eca247 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,5 +13,5 @@ itertools = "0.8.0"
 rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
 rustpython-parser = { path = "../parser", version = "0.1.1" }
 num-complex = { version = "0.2", features = ["serde"] }
-log = "0.3"
-arrayvec = "0.4"
+log = "0.4"
+arrayvec = "0.5"

From 407bb9e4cb3346a16b61f5390d6c985abd099f8b Mon Sep 17 00:00:00 2001
From: Seo Sanghyeon <sanxiyn@gmail.com>
Date: Fri, 11 Oct 2019 17:33:06 +0900
Subject: [PATCH 082/459] Implement IndentationError

---
 src/error.rs | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/src/error.rs b/src/error.rs
index 62546ddb..190d20b4 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -1,5 +1,6 @@
 use rustpython_parser::error::{LexicalErrorType, ParseError, ParseErrorType};
 use rustpython_parser::location::Location;
+use rustpython_parser::token::Tok;
 
 use std::error::Error;
 use std::fmt;
@@ -41,6 +42,26 @@ pub enum CompileErrorType {
 }
 
 impl CompileError {
+    pub fn is_indentation_error(&self) -> bool {
+        if let CompileErrorType::Parse(parse) = &self.error {
+            match parse {
+                ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true,
+                ParseErrorType::UnrecognizedToken(token, expected) => {
+                    if *token == Tok::Indent {
+                        true
+                    } else if expected.clone() == Some("Indent".to_string()) {
+                        true
+                    } else {
+                        false
+                    }
+                }
+                _ => false,
+            }
+        } else {
+            false
+        }
+    }
+
     pub fn is_tab_error(&self) -> bool {
         if let CompileErrorType::Parse(parse) = &self.error {
             if let ParseErrorType::Lexical(lex) = parse {

From f2b696a75b5f34ed560262d80f396fdd2265415d Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 28 Aug 2019 03:39:06 +0900
Subject: [PATCH 083/459] Add mode to ast::parse

---
 src/compile.rs | 31 +------------------------------
 src/lib.rs     |  1 +
 src/mode.rs    | 40 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 42 insertions(+), 30 deletions(-)
 create mode 100644 src/mode.rs

diff --git a/src/compile.rs b/src/compile.rs
index c457ae8e..b6f60219 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -6,6 +6,7 @@
 //!   https://github.com/micropython/micropython/blob/master/py/compile.c
 
 use crate::error::{CompileError, CompileErrorType};
+pub use crate::mode::Mode;
 use crate::output_stream::{CodeObjectStream, OutputStream};
 use crate::peephole::PeepholeOptimizer;
 use crate::symboltable::{
@@ -105,36 +106,6 @@ pub fn compile_program_single(
     })
 }
 
-#[derive(Clone, Copy)]
-pub enum Mode {
-    Exec,
-    Eval,
-    Single,
-}
-
-impl std::str::FromStr for Mode {
-    type Err = ModeParseError;
-    fn from_str(s: &str) -> Result<Self, ModeParseError> {
-        match s {
-            "exec" => Ok(Mode::Exec),
-            "eval" => Ok(Mode::Eval),
-            "single" => Ok(Mode::Single),
-            _ => Err(ModeParseError { _priv: () }),
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct ModeParseError {
-    _priv: (),
-}
-
-impl std::fmt::Display for ModeParseError {
-    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, r#"mode should be "exec", "eval", or "single""#)
-    }
-}
-
 impl<O> Default for Compiler<O>
 where
     O: OutputStream,
diff --git a/src/lib.rs b/src/lib.rs
index 9e18f1c2..4d469350 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -8,6 +8,7 @@ extern crate log;
 
 pub mod compile;
 pub mod error;
+pub mod mode;
 pub(crate) mod output_stream;
 pub mod peephole;
 pub mod symboltable;
diff --git a/src/mode.rs b/src/mode.rs
new file mode 100644
index 00000000..08137751
--- /dev/null
+++ b/src/mode.rs
@@ -0,0 +1,40 @@
+use rustpython_parser::parser;
+
+#[derive(Clone, Copy)]
+pub enum Mode {
+    Exec,
+    Eval,
+    Single,
+}
+
+impl std::str::FromStr for Mode {
+    type Err = ModeParseError;
+    fn from_str(s: &str) -> Result<Self, ModeParseError> {
+        match s {
+            "exec" => Ok(Mode::Exec),
+            "eval" => Ok(Mode::Eval),
+            "single" => Ok(Mode::Single),
+            _ => Err(ModeParseError { _priv: () }),
+        }
+    }
+}
+
+impl Mode {
+    pub fn to_parser_mode(self) -> parser::Mode {
+        match self {
+            Self::Exec | Self::Single => parser::Mode::Program,
+            Self::Eval => parser::Mode::Statement,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ModeParseError {
+    _priv: (),
+}
+
+impl std::fmt::Display for ModeParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, r#"mode should be "exec", "eval", or "single""#)
+    }
+}

From 0000903a7d13a3d3f6760cccb5398ba08a771c77 Mon Sep 17 00:00:00 2001
From: Seo Sanghyeon <sanxiyn@gmail.com>
Date: Sat, 12 Oct 2019 18:18:45 +0900
Subject: [PATCH 084/459] Shut up Clippy

---
 src/error.rs | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/error.rs b/src/error.rs
index 190d20b4..60ea7339 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -47,13 +47,7 @@ impl CompileError {
             match parse {
                 ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true,
                 ParseErrorType::UnrecognizedToken(token, expected) => {
-                    if *token == Tok::Indent {
-                        true
-                    } else if expected.clone() == Some("Indent".to_string()) {
-                        true
-                    } else {
-                        false
-                    }
+                    *token == Tok::Indent || expected.clone() == Some("Indent".to_string())
                 }
                 _ => false,
             }

From 939b49dc817a9ab37e13d98da5874e63a5b6c6c2 Mon Sep 17 00:00:00 2001
From: ChJR <chjr8815@gmail.com>
Date: Sun, 13 Oct 2019 15:53:49 +0900
Subject: [PATCH 085/459] Add build compatibility with rustc 1.36.0 since #1318
 merged

---
 src/mode.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mode.rs b/src/mode.rs
index 08137751..e6ca51f3 100644
--- a/src/mode.rs
+++ b/src/mode.rs
@@ -22,8 +22,8 @@ impl std::str::FromStr for Mode {
 impl Mode {
     pub fn to_parser_mode(self) -> parser::Mode {
         match self {
-            Self::Exec | Self::Single => parser::Mode::Program,
-            Self::Eval => parser::Mode::Statement,
+            Mode::Exec | Mode::Single => parser::Mode::Program,
+            Mode::Eval => parser::Mode::Statement,
         }
     }
 }

From 0fd098569e1fd9a56b49e002f0c4841c8de72c9a Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sun, 13 Oct 2019 22:43:11 -0500
Subject: [PATCH 086/459] Change FunctionOpArg to CodeFlags, stored in
 CodeObject

---
 src/compile.rs       | 51 +++++++++++++++++++++-----------------------
 src/output_stream.rs |  4 ++--
 2 files changed, 26 insertions(+), 29 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index b6f60219..ca185e66 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -137,6 +137,7 @@ impl<O: OutputStream> Compiler<O> {
     fn push_new_code_object(&mut self, obj_name: String) {
         let line_number = self.get_source_line_number();
         self.push_output(CodeObject::new(
+            Default::default(),
             Vec::new(),
             Varargs::None,
             Vec::new(),
@@ -595,11 +596,7 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
-    fn enter_function(
-        &mut self,
-        name: &str,
-        args: &ast::Parameters,
-    ) -> Result<bytecode::FunctionOpArg, CompileError> {
+    fn enter_function(&mut self, name: &str, args: &ast::Parameters) -> Result<(), CompileError> {
         let have_defaults = !args.defaults.is_empty();
         if have_defaults {
             // Construct a tuple:
@@ -633,8 +630,17 @@ impl<O: OutputStream> Compiler<O> {
             });
         }
 
+        let mut flags = bytecode::CodeFlags::default();
+        if have_defaults {
+            flags |= bytecode::CodeFlags::HAS_DEFAULTS;
+        }
+        if num_kw_only_defaults > 0 {
+            flags |= bytecode::CodeFlags::HAS_KW_ONLY_DEFAULTS;
+        }
+
         let line_number = self.get_source_line_number();
         self.push_output(CodeObject::new(
+            flags,
             args.args.iter().map(|a| a.arg.clone()).collect(),
             compile_varargs(&args.vararg),
             args.kwonlyargs.iter().map(|a| a.arg.clone()).collect(),
@@ -645,15 +651,7 @@ impl<O: OutputStream> Compiler<O> {
         ));
         self.enter_scope();
 
-        let mut flags = bytecode::FunctionOpArg::default();
-        if have_defaults {
-            flags |= bytecode::FunctionOpArg::HAS_DEFAULTS;
-        }
-        if num_kw_only_defaults > 0 {
-            flags |= bytecode::FunctionOpArg::HAS_KW_ONLY_DEFAULTS;
-        }
-
-        Ok(flags)
+        Ok(())
     }
 
     fn prepare_decorators(
@@ -813,7 +811,7 @@ impl<O: OutputStream> Compiler<O> {
 
         self.prepare_decorators(decorator_list)?;
 
-        let mut flags = self.enter_function(name, args)?;
+        self.enter_function(name, args)?;
 
         let (body, doc_str) = get_doc(body);
 
@@ -832,7 +830,7 @@ impl<O: OutputStream> Compiler<O> {
             }
         }
 
-        let code = self.pop_code_object();
+        let mut code = self.pop_code_object();
         self.leave_scope();
 
         // Prepare type annotations:
@@ -864,7 +862,7 @@ impl<O: OutputStream> Compiler<O> {
         }
 
         if num_annotations > 0 {
-            flags |= bytecode::FunctionOpArg::HAS_ANNOTATIONS;
+            code.flags |= bytecode::CodeFlags::HAS_ANNOTATIONS;
             self.emit(Instruction::BuildMap {
                 size: num_annotations,
                 unpack: false,
@@ -884,7 +882,7 @@ impl<O: OutputStream> Compiler<O> {
         });
 
         // Turn code object into function object:
-        self.emit(Instruction::MakeFunction { flags });
+        self.emit(Instruction::MakeFunction);
         self.store_docstring(doc_str);
         self.apply_decorators(decorator_list);
 
@@ -915,6 +913,7 @@ impl<O: OutputStream> Compiler<O> {
         self.emit(Instruction::LoadBuildClass);
         let line_number = self.get_source_line_number();
         self.push_output(CodeObject::new(
+            Default::default(),
             vec![],
             Varargs::None,
             vec![],
@@ -950,7 +949,8 @@ impl<O: OutputStream> Compiler<O> {
         });
         self.emit(Instruction::ReturnValue);
 
-        let code = self.pop_code_object();
+        let mut code = self.pop_code_object();
+        code.flags &= !bytecode::CodeFlags::NEW_LOCALS;
         self.leave_scope();
 
         self.emit(Instruction::LoadConst {
@@ -965,9 +965,7 @@ impl<O: OutputStream> Compiler<O> {
         });
 
         // Turn code object into function object:
-        self.emit(Instruction::MakeFunction {
-            flags: bytecode::FunctionOpArg::default() & !bytecode::FunctionOpArg::NEW_LOCALS,
-        });
+        self.emit(Instruction::MakeFunction);
 
         self.emit(Instruction::LoadConst {
             value: bytecode::Constant::String {
@@ -1615,7 +1613,7 @@ impl<O: OutputStream> Compiler<O> {
             Lambda { args, body } => {
                 let name = "<lambda>".to_string();
                 // no need to worry about the self.loop_depth because there are no loops in lambda expressions
-                let flags = self.enter_function(&name, args)?;
+                self.enter_function(&name, args)?;
                 self.compile_expression(body)?;
                 self.emit(Instruction::ReturnValue);
                 let code = self.pop_code_object();
@@ -1629,7 +1627,7 @@ impl<O: OutputStream> Compiler<O> {
                     value: bytecode::Constant::String { value: name },
                 });
                 // Turn code object into function object:
-                self.emit(Instruction::MakeFunction { flags });
+                self.emit(Instruction::MakeFunction);
             }
             Comprehension { kind, generators } => {
                 self.compile_comprehension(kind, generators)?;
@@ -1810,6 +1808,7 @@ impl<O: OutputStream> Compiler<O> {
         let line_number = self.get_source_line_number();
         // Create magnificent function <listcomp>:
         self.push_output(CodeObject::new(
+            Default::default(),
             vec![".0".to_string()],
             Varargs::None,
             vec![],
@@ -1945,9 +1944,7 @@ impl<O: OutputStream> Compiler<O> {
         });
 
         // Turn code object into function object:
-        self.emit(Instruction::MakeFunction {
-            flags: bytecode::FunctionOpArg::default(),
-        });
+        self.emit(Instruction::MakeFunction);
 
         // Evaluate iterated item:
         self.compile_expression(&generators[0].iter)?;
diff --git a/src/output_stream.rs b/src/output_stream.rs
index 9d0ece09..fa358817 100644
--- a/src/output_stream.rs
+++ b/src/output_stream.rs
@@ -1,4 +1,4 @@
-use rustpython_bytecode::bytecode::{CodeObject, Instruction, Label, Location};
+use rustpython_bytecode::bytecode::{CodeFlags, CodeObject, Instruction, Label, Location};
 
 pub trait OutputStream: From<CodeObject> + Into<CodeObject> {
     /// Output an instruction
@@ -34,6 +34,6 @@ impl OutputStream for CodeObjectStream {
         self.code.label_map.insert(label, position);
     }
     fn mark_generator(&mut self) {
-        self.code.is_generator = true;
+        self.code.flags |= CodeFlags::IS_GENERATOR;
     }
 }

From 142a29be4adfcb228717d2464d211467ed3f4841 Mon Sep 17 00:00:00 2001
From: ichyo <ichyo00@gmail.com>
Date: Thu, 24 Oct 2019 00:26:02 +0900
Subject: [PATCH 087/459] Fix wrong const optimization for "1.0 / 0.0" case

In cpython, "1.0 / 0.0" raises ZeroDivisionError
```
>>> 1.0 / 0.0
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
ZeroDivisionError: float division by zer
```

but RustPython doesn't
```
>>>>> 1.0 / 0.0
inf
```

This is because they emits different byte codes.

```
$ echo "1.0 / 0.0" > zero-div.py

$ python -m dis zero-div.py
  1           0 LOAD_CONST               0 (1.0)
              2 LOAD_CONST               1 (0.0)
              4 BINARY_TRUE_DIVIDE
              6 POP_TOP
              8 LOAD_CONST               2 (None)
             10 RETURN_VALUE

$ cargo run --example dis zero-div.py
zero-div.py:
                 0 LoadConst            (inf)
                 1 Pop
                 2 LoadConst            (None)
                 3 ReturnValue
```

This commit stops optimization for zero division case
to generate same byte codes.
---
 src/peephole/optimizations.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/peephole/optimizations.rs b/src/peephole/optimizations.rs
index b0fb7531..28bac862 100644
--- a/src/peephole/optimizations.rs
+++ b/src/peephole/optimizations.rs
@@ -52,7 +52,7 @@ pub fn operator(buf: &mut impl OptimizationBuffer) {
             (op!(Multiply), lc!(Float, lhs), lc!(Float, rhs)) => {
                 emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs * rhs)
             }
-            (op!(Divide), lc!(Float, lhs), lc!(Float, rhs)) => {
+            (op!(Divide), lc!(Float, lhs), lc!(Float, rhs)) if rhs != 0.0 => {
                 emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs / rhs)
             }
             (op!(Power), lc!(Float, lhs), lc!(Float, rhs)) => {

From 30ab196fdc73390a3cd6d8d81c0e7638fac5b75a Mon Sep 17 00:00:00 2001
From: joshua1b <joshua1b2823@gmail.com>
Date: Sat, 2 Nov 2019 23:47:44 +0900
Subject: [PATCH 088/459] Set docstring of function as None if not declared

In cpython, if there is no docstring declared in function definition,
the `__doc__` attribute of the function is None.
So this implements the behavior.

Fix #1523
---
 src/compile.rs | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index ca185e66..ac4f00a6 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1015,22 +1015,23 @@ impl<O: OutputStream> Compiler<O> {
     }
 
     fn store_docstring(&mut self, doc_str: Option<String>) {
-        if let Some(doc_string) = doc_str {
-            // Duplicate top of stack (the function or class object)
-            self.emit(Instruction::Duplicate);
+        // Duplicate top of stack (the function or class object)
+        self.emit(Instruction::Duplicate);
 
-            // Doc string value:
-            self.emit(Instruction::LoadConst {
-                value: bytecode::Constant::String {
-                    value: doc_string.to_string(),
+        // Doc string value:
+        self.emit(Instruction::LoadConst {
+            value: match doc_str {
+                Some(doc) => bytecode::Constant::String {
+                    value: doc.to_string(),
                 },
-            });
+                None => bytecode::Constant::None, // set docstring None if not declared
+            },
+        });
 
-            self.emit(Instruction::Rotate { amount: 2 });
-            self.emit(Instruction::StoreAttr {
-                name: "__doc__".to_string(),
-            });
-        }
+        self.emit(Instruction::Rotate { amount: 2 });
+        self.emit(Instruction::StoreAttr {
+            name: "__doc__".to_string(),
+        });
     }
 
     fn compile_while(

From 3638591cdedff5c98c79718cf95350a7a38a0824 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 16 Oct 2019 20:07:30 -0500
Subject: [PATCH 089/459] Add coroutines, async/await functionality, and
 gen.close()

---
 src/compile.rs | 40 +++++++++++++++++++++++++++++++---------
 1 file changed, 31 insertions(+), 9 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index ac4f00a6..f467cc5c 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -29,6 +29,7 @@ struct Compiler<O: OutputStream = BasicOutputStream> {
     current_qualified_path: Option<String>,
     in_loop: bool,
     in_function_def: bool,
+    in_async_func: bool,
     optimize: u8,
 }
 
@@ -126,6 +127,7 @@ impl<O: OutputStream> Compiler<O> {
             current_qualified_path: None,
             in_loop: false,
             in_function_def: false,
+            in_async_func: false,
             optimize,
         }
     }
@@ -451,11 +453,7 @@ impl<O: OutputStream> Compiler<O> {
                 decorator_list,
                 returns,
             } => {
-                if *is_async {
-                    unimplemented!("async def");
-                } else {
-                    self.compile_function_def(name, args, body, decorator_list, returns)?
-                }
+                self.compile_function_def(name, args, body, decorator_list, returns, *is_async)?;
             }
             ClassDef {
                 name,
@@ -797,11 +795,15 @@ impl<O: OutputStream> Compiler<O> {
         body: &[ast::Statement],
         decorator_list: &[ast::Expression],
         returns: &Option<ast::Expression>, // TODO: use type hint somehow..
+        is_async: bool,
     ) -> Result<(), CompileError> {
         // Create bytecode for this function:
         // remember to restore self.in_loop to the original after the function is compiled
         let was_in_loop = self.in_loop;
         let was_in_function_def = self.in_function_def;
+
+        let was_in_async_func = self.in_async_func;
+        self.in_async_func = is_async;
         self.in_loop = false;
         self.in_function_def = true;
 
@@ -870,6 +872,10 @@ impl<O: OutputStream> Compiler<O> {
             });
         }
 
+        if is_async {
+            code.flags |= bytecode::CodeFlags::IS_COROUTINE;
+        }
+
         self.emit(Instruction::LoadConst {
             value: bytecode::Constant::Code {
                 code: Box::new(code),
@@ -891,6 +897,7 @@ impl<O: OutputStream> Compiler<O> {
         self.current_qualified_path = old_qualified_path;
         self.in_loop = was_in_loop;
         self.in_function_def = was_in_function_def;
+        self.in_async_func = was_in_async_func;
         Ok(())
     }
 
@@ -1551,7 +1558,7 @@ impl<O: OutputStream> Compiler<O> {
                 self.emit(Instruction::BuildSlice { size });
             }
             Yield { value } => {
-                if !self.in_function_def {
+                if !self.in_function_def || self.in_async_func {
                     return Err(CompileError {
                         error: CompileErrorType::InvalidYield,
                         location: self.current_source_location.clone(),
@@ -1566,8 +1573,13 @@ impl<O: OutputStream> Compiler<O> {
                 };
                 self.emit(Instruction::YieldValue);
             }
-            Await { .. } => {
-                unimplemented!("await");
+            Await { value } => {
+                self.compile_expression(value)?;
+                self.emit(Instruction::GetAwaitable);
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::None,
+                });
+                self.emit(Instruction::YieldFrom);
             }
             YieldFrom { value } => {
                 self.mark_generator();
@@ -1612,8 +1624,14 @@ impl<O: OutputStream> Compiler<O> {
                 self.load_name(name);
             }
             Lambda { args, body } => {
+                let was_in_loop = self.in_loop;
+                let was_in_function_def = self.in_function_def;
+                let was_in_async_func = self.in_async_func;
+                self.in_async_func = false;
+                self.in_loop = false;
+                self.in_function_def = true;
+
                 let name = "<lambda>".to_string();
-                // no need to worry about the self.loop_depth because there are no loops in lambda expressions
                 self.enter_function(&name, args)?;
                 self.compile_expression(body)?;
                 self.emit(Instruction::ReturnValue);
@@ -1629,6 +1647,10 @@ impl<O: OutputStream> Compiler<O> {
                 });
                 // Turn code object into function object:
                 self.emit(Instruction::MakeFunction);
+
+                self.in_loop = was_in_loop;
+                self.in_function_def = was_in_function_def;
+                self.in_async_func = was_in_async_func;
             }
             Comprehension { kind, generators } => {
                 self.compile_comprehension(kind, generators)?;

From ae825125a5fa22f31a5179c90059bfcecd42545b Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 16 Oct 2019 22:28:49 -0500
Subject: [PATCH 090/459] Add async with, reorganize how with blocks work

---
 src/compile.rs | 77 +++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 61 insertions(+), 16 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index f467cc5c..fae828dc 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -386,27 +386,72 @@ impl<O: OutputStream> Compiler<O> {
                 body,
             } => {
                 if *is_async {
-                    unimplemented!("async with");
-                } else {
-                    let end_label = self.new_label();
-                    for item in items {
-                        self.compile_expression(&item.context_expr)?;
-                        self.emit(Instruction::SetupWith { end: end_label });
-                        match &item.optional_vars {
-                            Some(var) => {
-                                self.compile_store(var)?;
-                            }
-                            None => {
-                                self.emit(Instruction::Pop);
+                    let end_labels = items
+                        .iter()
+                        .map(|item| {
+                            let end_label = self.new_label();
+                            self.compile_expression(&item.context_expr)?;
+                            self.emit(Instruction::BeforeAsyncWith);
+                            self.emit(Instruction::GetAwaitable);
+                            self.emit(Instruction::LoadConst {
+                                value: bytecode::Constant::None,
+                            });
+                            self.emit(Instruction::YieldFrom);
+                            self.emit(Instruction::SetupAsyncWith { end: end_label });
+                            match &item.optional_vars {
+                                Some(var) => {
+                                    self.compile_store(var)?;
+                                }
+                                None => {
+                                    self.emit(Instruction::Pop);
+                                }
                             }
-                        }
+                            Ok(end_label)
+                        })
+                        .collect::<Result<Vec<_>, CompileError>>()?;
+
+                    self.compile_statements(body)?;
+
+                    for end_label in end_labels {
+                        self.emit(Instruction::PopBlock);
+                        self.emit(Instruction::EnterFinally);
+                        self.set_label(end_label);
+                        self.emit(Instruction::WithCleanupStart);
+                        self.emit(Instruction::GetAwaitable);
+                        self.emit(Instruction::LoadConst {
+                            value: bytecode::Constant::None,
+                        });
+                        self.emit(Instruction::YieldFrom);
+                        self.emit(Instruction::WithCleanupFinish);
                     }
+                } else {
+                    let end_labels = items
+                        .iter()
+                        .map(|item| {
+                            let end_label = self.new_label();
+                            self.compile_expression(&item.context_expr)?;
+                            self.emit(Instruction::SetupWith { end: end_label });
+                            match &item.optional_vars {
+                                Some(var) => {
+                                    self.compile_store(var)?;
+                                }
+                                None => {
+                                    self.emit(Instruction::Pop);
+                                }
+                            }
+                            Ok(end_label)
+                        })
+                        .collect::<Result<Vec<_>, CompileError>>()?;
 
                     self.compile_statements(body)?;
-                    for _ in 0..items.len() {
-                        self.emit(Instruction::CleanupWith { end: end_label });
+
+                    for end_label in end_labels {
+                        self.emit(Instruction::PopBlock);
+                        self.emit(Instruction::EnterFinally);
+                        self.set_label(end_label);
+                        self.emit(Instruction::WithCleanupStart);
+                        self.emit(Instruction::WithCleanupFinish);
                     }
-                    self.set_label(end_label);
                 }
             }
             For {

From 19f62b97a39e2f07dbd8e8d90a3055210e8e85f8 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Thu, 17 Oct 2019 11:36:23 -0500
Subject: [PATCH 091/459] Deduplicate async/normal `with` compilation code

---
 src/compile.rs | 82 ++++++++++++++++++++------------------------------
 1 file changed, 32 insertions(+), 50 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index fae828dc..345a04d8 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -385,12 +385,15 @@ impl<O: OutputStream> Compiler<O> {
                 items,
                 body,
             } => {
-                if *is_async {
-                    let end_labels = items
-                        .iter()
-                        .map(|item| {
-                            let end_label = self.new_label();
-                            self.compile_expression(&item.context_expr)?;
+                let is_async = *is_async;
+
+                let end_labels = items
+                    .iter()
+                    .map(|item| {
+                        let end_label = self.new_label();
+                        self.compile_expression(&item.context_expr)?;
+
+                        if is_async {
                             self.emit(Instruction::BeforeAsyncWith);
                             self.emit(Instruction::GetAwaitable);
                             self.emit(Instruction::LoadConst {
@@ -398,60 +401,39 @@ impl<O: OutputStream> Compiler<O> {
                             });
                             self.emit(Instruction::YieldFrom);
                             self.emit(Instruction::SetupAsyncWith { end: end_label });
-                            match &item.optional_vars {
-                                Some(var) => {
-                                    self.compile_store(var)?;
-                                }
-                                None => {
-                                    self.emit(Instruction::Pop);
-                                }
+                        } else {
+                            self.emit(Instruction::SetupWith { end: end_label });
+                        }
+
+                        match &item.optional_vars {
+                            Some(var) => {
+                                self.compile_store(var)?;
                             }
-                            Ok(end_label)
-                        })
-                        .collect::<Result<Vec<_>, CompileError>>()?;
+                            None => {
+                                self.emit(Instruction::Pop);
+                            }
+                        }
+                        Ok(end_label)
+                    })
+                    .collect::<Result<Vec<_>, CompileError>>()?;
 
-                    self.compile_statements(body)?;
+                self.compile_statements(body)?;
 
-                    for end_label in end_labels {
-                        self.emit(Instruction::PopBlock);
-                        self.emit(Instruction::EnterFinally);
-                        self.set_label(end_label);
-                        self.emit(Instruction::WithCleanupStart);
+                for end_label in end_labels {
+                    self.emit(Instruction::PopBlock);
+                    self.emit(Instruction::EnterFinally);
+                    self.set_label(end_label);
+                    self.emit(Instruction::WithCleanupStart);
+
+                    if is_async {
                         self.emit(Instruction::GetAwaitable);
                         self.emit(Instruction::LoadConst {
                             value: bytecode::Constant::None,
                         });
                         self.emit(Instruction::YieldFrom);
-                        self.emit(Instruction::WithCleanupFinish);
                     }
-                } else {
-                    let end_labels = items
-                        .iter()
-                        .map(|item| {
-                            let end_label = self.new_label();
-                            self.compile_expression(&item.context_expr)?;
-                            self.emit(Instruction::SetupWith { end: end_label });
-                            match &item.optional_vars {
-                                Some(var) => {
-                                    self.compile_store(var)?;
-                                }
-                                None => {
-                                    self.emit(Instruction::Pop);
-                                }
-                            }
-                            Ok(end_label)
-                        })
-                        .collect::<Result<Vec<_>, CompileError>>()?;
-
-                    self.compile_statements(body)?;
 
-                    for end_label in end_labels {
-                        self.emit(Instruction::PopBlock);
-                        self.emit(Instruction::EnterFinally);
-                        self.set_label(end_label);
-                        self.emit(Instruction::WithCleanupStart);
-                        self.emit(Instruction::WithCleanupFinish);
-                    }
+                    self.emit(Instruction::WithCleanupFinish);
                 }
             }
             For {

From c39e0baab72073f9de7db166adb3d402883c6319 Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Thu, 17 Oct 2019 23:34:43 -0500
Subject: [PATCH 092/459] Add async for

---
 src/compile.rs | 80 +++++++++++++++++++++++++++++++++++---------------
 1 file changed, 56 insertions(+), 24 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 345a04d8..e013f597 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -442,13 +442,7 @@ impl<O: OutputStream> Compiler<O> {
                 iter,
                 body,
                 orelse,
-            } => {
-                if *is_async {
-                    unimplemented!("async for");
-                } else {
-                    self.compile_for(target, iter, body, orelse)?
-                }
-            }
+            } => self.compile_for(target, iter, body, orelse, *is_async)?,
             Raise { exception, cause } => match exception {
                 Some(value) => {
                     self.compile_expression(value)?;
@@ -736,14 +730,9 @@ impl<O: OutputStream> Compiler<O> {
                 self.emit(Instruction::Duplicate);
 
                 // Check exception type:
-                self.emit(Instruction::LoadName {
-                    name: String::from("isinstance"),
-                    scope: bytecode::NameScope::Global,
-                });
-                self.emit(Instruction::Rotate { amount: 2 });
                 self.compile_expression(exc_type)?;
-                self.emit(Instruction::CallFunction {
-                    typ: CallType::Positional(2),
+                self.emit(Instruction::CompareOperation {
+                    op: bytecode::ComparisonOperator::ExceptionMatch,
                 });
 
                 // We cannot handle this exception type:
@@ -1108,11 +1097,13 @@ impl<O: OutputStream> Compiler<O> {
         iter: &ast::Expression,
         body: &[ast::Statement],
         orelse: &Option<Vec<ast::Statement>>,
+        is_async: bool,
     ) -> Result<(), CompileError> {
         // Start loop
         let start_label = self.new_label();
         let else_label = self.new_label();
         let end_label = self.new_label();
+
         self.emit(Instruction::SetupLoop {
             start: start_label,
             end: end_label,
@@ -1121,19 +1112,57 @@ impl<O: OutputStream> Compiler<O> {
         // The thing iterated:
         self.compile_expression(iter)?;
 
-        // Retrieve Iterator
-        self.emit(Instruction::GetIter);
+        if is_async {
+            let check_asynciter_label = self.new_label();
+            let body_label = self.new_label();
 
-        self.set_label(start_label);
-        self.emit(Instruction::ForIter { target: else_label });
+            self.emit(Instruction::GetAIter);
 
-        // Start of loop iteration, set targets:
-        self.compile_store(target)?;
+            self.set_label(start_label);
+            self.emit(Instruction::SetupExcept {
+                handler: check_asynciter_label,
+            });
+            self.emit(Instruction::GetANext);
+            self.emit(Instruction::LoadConst {
+                value: bytecode::Constant::None,
+            });
+            self.emit(Instruction::YieldFrom);
+            self.compile_store(target)?;
+            self.emit(Instruction::PopBlock);
+            self.emit(Instruction::Jump { target: body_label });
 
-        let was_in_loop = self.in_loop;
-        self.in_loop = true;
-        self.compile_statements(body)?;
-        self.in_loop = was_in_loop;
+            self.set_label(check_asynciter_label);
+            self.emit(Instruction::Duplicate);
+            self.emit(Instruction::LoadName {
+                name: "StopAsyncIteration".to_string(),
+                scope: bytecode::NameScope::Global,
+            });
+            self.emit(Instruction::CompareOperation {
+                op: bytecode::ComparisonOperator::ExceptionMatch,
+            });
+            self.emit(Instruction::JumpIfTrue { target: else_label });
+            self.emit(Instruction::Raise { argc: 0 });
+
+            let was_in_loop = self.in_loop;
+            self.in_loop = true;
+            self.set_label(body_label);
+            self.compile_statements(body)?;
+            self.in_loop = was_in_loop;
+        } else {
+            // Retrieve Iterator
+            self.emit(Instruction::GetIter);
+
+            self.set_label(start_label);
+            self.emit(Instruction::ForIter { target: else_label });
+
+            // Start of loop iteration, set targets:
+            self.compile_store(target)?;
+
+            let was_in_loop = self.in_loop;
+            self.in_loop = true;
+            self.compile_statements(body)?;
+            self.in_loop = was_in_loop;
+        }
 
         self.emit(Instruction::Jump {
             target: start_label,
@@ -1144,6 +1173,9 @@ impl<O: OutputStream> Compiler<O> {
             self.compile_statements(orelse)?;
         }
         self.set_label(end_label);
+        if is_async {
+            self.emit(Instruction::Pop);
+        }
         Ok(())
     }
 

From a51a3e0e4b1e717e484c5ff9dd3a7cec98af289e Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 1 Nov 2019 03:31:23 +0000
Subject: [PATCH 093/459] Add a CompileContext

---
 src/compile.rs | 108 +++++++++++++++++++++++++++++--------------------
 1 file changed, 65 insertions(+), 43 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index e013f597..c2400d6b 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -27,12 +27,32 @@ struct Compiler<O: OutputStream = BasicOutputStream> {
     source_path: Option<String>,
     current_source_location: ast::Location,
     current_qualified_path: Option<String>,
-    in_loop: bool,
-    in_function_def: bool,
-    in_async_func: bool,
+    ctx: CompileContext,
     optimize: u8,
 }
 
+#[derive(Clone, Copy)]
+struct CompileContext {
+    in_loop: bool,
+    func: FunctionContext,
+}
+
+#[derive(Clone, Copy)]
+enum FunctionContext {
+    NoFunction,
+    Function,
+    AsyncFunction,
+}
+
+impl CompileContext {
+    fn in_func(self) -> bool {
+        match self.func {
+            FunctionContext::NoFunction => false,
+            _ => true,
+        }
+    }
+}
+
 /// Compile a given sourcecode into a bytecode object.
 pub fn compile(
     source: &str,
@@ -125,9 +145,10 @@ impl<O: OutputStream> Compiler<O> {
             source_path: None,
             current_source_location: ast::Location::default(),
             current_qualified_path: None,
-            in_loop: false,
-            in_function_def: false,
-            in_async_func: false,
+            ctx: CompileContext {
+                in_loop: false,
+                func: FunctionContext::NoFunction,
+            },
             optimize,
         }
     }
@@ -510,7 +531,7 @@ impl<O: OutputStream> Compiler<O> {
                 }
             }
             Break => {
-                if !self.in_loop {
+                if !self.ctx.in_loop {
                     return Err(CompileError {
                         error: CompileErrorType::InvalidBreak,
                         location: statement.location.clone(),
@@ -519,7 +540,7 @@ impl<O: OutputStream> Compiler<O> {
                 self.emit(Instruction::Break);
             }
             Continue => {
-                if !self.in_loop {
+                if !self.ctx.in_loop {
                     return Err(CompileError {
                         error: CompileErrorType::InvalidContinue,
                         location: statement.location.clone(),
@@ -528,7 +549,7 @@ impl<O: OutputStream> Compiler<O> {
                 self.emit(Instruction::Continue);
             }
             Return { value } => {
-                if !self.in_function_def {
+                if !self.ctx.in_func() {
                     return Err(CompileError {
                         error: CompileErrorType::InvalidReturn,
                         location: statement.location.clone(),
@@ -814,17 +835,20 @@ impl<O: OutputStream> Compiler<O> {
         is_async: bool,
     ) -> Result<(), CompileError> {
         // Create bytecode for this function:
-        // remember to restore self.in_loop to the original after the function is compiled
-        let was_in_loop = self.in_loop;
-        let was_in_function_def = self.in_function_def;
+        // remember to restore self.ctx.in_loop to the original after the function is compiled
+        let prev_ctx = self.ctx;
 
-        let was_in_async_func = self.in_async_func;
-        self.in_async_func = is_async;
-        self.in_loop = false;
-        self.in_function_def = true;
+        self.ctx = CompileContext {
+            in_loop: false,
+            func: if is_async {
+                FunctionContext::AsyncFunction
+            } else {
+                FunctionContext::Function
+            },
+        };
 
-        let old_qualified_path = self.current_qualified_path.clone();
         let qualified_name = self.create_qualified_name(name, "");
+        let old_qualified_path = self.current_qualified_path.take();
         self.current_qualified_path = Some(self.create_qualified_name(name, ".<locals>"));
 
         self.prepare_decorators(decorator_list)?;
@@ -911,9 +935,7 @@ impl<O: OutputStream> Compiler<O> {
         self.store_name(name);
 
         self.current_qualified_path = old_qualified_path;
-        self.in_loop = was_in_loop;
-        self.in_function_def = was_in_function_def;
-        self.in_async_func = was_in_async_func;
+        self.ctx = prev_ctx;
         Ok(())
     }
 
@@ -925,11 +947,14 @@ impl<O: OutputStream> Compiler<O> {
         keywords: &[ast::Keyword],
         decorator_list: &[ast::Expression],
     ) -> Result<(), CompileError> {
-        let was_in_loop = self.in_loop;
-        self.in_loop = false;
+        let prev_ctx = self.ctx;
+        self.ctx = CompileContext {
+            func: FunctionContext::NoFunction,
+            in_loop: false,
+        };
 
-        let old_qualified_path = self.current_qualified_path.clone();
         let qualified_name = self.create_qualified_name(name, "");
+        let old_qualified_path = self.current_qualified_path.take();
         self.current_qualified_path = Some(qualified_name.clone());
 
         self.prepare_decorators(decorator_list)?;
@@ -1033,7 +1058,7 @@ impl<O: OutputStream> Compiler<O> {
 
         self.store_name(name);
         self.current_qualified_path = old_qualified_path;
-        self.in_loop = was_in_loop;
+        self.ctx = prev_ctx;
         Ok(())
     }
 
@@ -1075,10 +1100,10 @@ impl<O: OutputStream> Compiler<O> {
 
         self.compile_jump_if(test, false, else_label)?;
 
-        let was_in_loop = self.in_loop;
-        self.in_loop = true;
+        let was_in_loop = self.ctx.in_loop;
+        self.ctx.in_loop = true;
         self.compile_statements(body)?;
-        self.in_loop = was_in_loop;
+        self.ctx.in_loop = was_in_loop;
         self.emit(Instruction::Jump {
             target: start_label,
         });
@@ -1143,11 +1168,11 @@ impl<O: OutputStream> Compiler<O> {
             self.emit(Instruction::JumpIfTrue { target: else_label });
             self.emit(Instruction::Raise { argc: 0 });
 
-            let was_in_loop = self.in_loop;
-            self.in_loop = true;
+            let was_in_loop = self.ctx.in_loop;
+            self.ctx.in_loop = true;
             self.set_label(body_label);
             self.compile_statements(body)?;
-            self.in_loop = was_in_loop;
+            self.ctx.in_loop = was_in_loop;
         } else {
             // Retrieve Iterator
             self.emit(Instruction::GetIter);
@@ -1158,10 +1183,10 @@ impl<O: OutputStream> Compiler<O> {
             // Start of loop iteration, set targets:
             self.compile_store(target)?;
 
-            let was_in_loop = self.in_loop;
-            self.in_loop = true;
+            let was_in_loop = self.ctx.in_loop;
+            self.ctx.in_loop = true;
             self.compile_statements(body)?;
-            self.in_loop = was_in_loop;
+            self.ctx.in_loop = was_in_loop;
         }
 
         self.emit(Instruction::Jump {
@@ -1617,7 +1642,7 @@ impl<O: OutputStream> Compiler<O> {
                 self.emit(Instruction::BuildSlice { size });
             }
             Yield { value } => {
-                if !self.in_function_def || self.in_async_func {
+                if !self.ctx.in_func() {
                     return Err(CompileError {
                         error: CompileErrorType::InvalidYield,
                         location: self.current_source_location.clone(),
@@ -1683,12 +1708,11 @@ impl<O: OutputStream> Compiler<O> {
                 self.load_name(name);
             }
             Lambda { args, body } => {
-                let was_in_loop = self.in_loop;
-                let was_in_function_def = self.in_function_def;
-                let was_in_async_func = self.in_async_func;
-                self.in_async_func = false;
-                self.in_loop = false;
-                self.in_function_def = true;
+                let prev_ctx = self.ctx;
+                self.ctx = CompileContext {
+                    in_loop: false,
+                    func: FunctionContext::Function,
+                };
 
                 let name = "<lambda>".to_string();
                 self.enter_function(&name, args)?;
@@ -1707,9 +1731,7 @@ impl<O: OutputStream> Compiler<O> {
                 // Turn code object into function object:
                 self.emit(Instruction::MakeFunction);
 
-                self.in_loop = was_in_loop;
-                self.in_function_def = was_in_function_def;
-                self.in_async_func = was_in_async_func;
+                self.ctx = prev_ctx;
             }
             Comprehension { kind, generators } => {
                 self.compile_comprehension(kind, generators)?;

From 28fbdffdc8dbf668414aef9f4a30a7bc9b851606 Mon Sep 17 00:00:00 2001
From: yanganto <yanganto@gmail.com>
Date: Thu, 21 Nov 2019 17:04:32 +0800
Subject: [PATCH 094/459] Provide caret diagnostics for SyntaxError

visualize syntax error with caret diagnostics in shell, eval, exec,
when the error statement and error location are provided.
---
 src/compile.rs     |  9 +++++++++
 src/error.rs       | 46 ++++++++++++++++++++++++++++++++--------------
 src/symboltable.rs |  1 +
 3 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index c2400d6b..e4bf2084 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -255,6 +255,7 @@ impl<O: OutputStream> Compiler<O> {
                 self.compile_expression(expression)?;
             } else {
                 return Err(CompileError {
+                    statement: None,
                     error: CompileErrorType::ExpectExpr,
                     location: statement.location.clone(),
                 });
@@ -533,6 +534,7 @@ impl<O: OutputStream> Compiler<O> {
             Break => {
                 if !self.ctx.in_loop {
                     return Err(CompileError {
+                        statement: None,
                         error: CompileErrorType::InvalidBreak,
                         location: statement.location.clone(),
                     });
@@ -542,6 +544,7 @@ impl<O: OutputStream> Compiler<O> {
             Continue => {
                 if !self.ctx.in_loop {
                     return Err(CompileError {
+                        statement: None,
                         error: CompileErrorType::InvalidContinue,
                         location: statement.location.clone(),
                     });
@@ -551,6 +554,7 @@ impl<O: OutputStream> Compiler<O> {
             Return { value } => {
                 if !self.ctx.in_func() {
                     return Err(CompileError {
+                        statement: None,
                         error: CompileErrorType::InvalidReturn,
                         location: statement.location.clone(),
                     });
@@ -628,6 +632,7 @@ impl<O: OutputStream> Compiler<O> {
             }
             _ => {
                 return Err(CompileError {
+                    statement: None,
                     error: CompileErrorType::Delete(expression.name()),
                     location: self.current_source_location.clone(),
                 });
@@ -1331,6 +1336,7 @@ impl<O: OutputStream> Compiler<O> {
                     if let ast::ExpressionType::Starred { .. } = &element.node {
                         if seen_star {
                             return Err(CompileError {
+                                statement: None,
                                 error: CompileErrorType::StarArgs,
                                 location: self.current_source_location.clone(),
                             });
@@ -1360,6 +1366,7 @@ impl<O: OutputStream> Compiler<O> {
             }
             _ => {
                 return Err(CompileError {
+                    statement: None,
                     error: CompileErrorType::Assign(target.name()),
                     location: self.current_source_location.clone(),
                 });
@@ -1644,6 +1651,7 @@ impl<O: OutputStream> Compiler<O> {
             Yield { value } => {
                 if !self.ctx.in_func() {
                     return Err(CompileError {
+                        statement: Option::None,
                         error: CompileErrorType::InvalidYield,
                         location: self.current_source_location.clone(),
                     });
@@ -1738,6 +1746,7 @@ impl<O: OutputStream> Compiler<O> {
             }
             Starred { .. } => {
                 return Err(CompileError {
+                    statement: Option::None,
                     error: CompileErrorType::SyntaxError(std::string::String::from(
                         "Invalid starred expression",
                     )),
diff --git a/src/error.rs b/src/error.rs
index 60ea7339..346ff3ac 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -7,13 +7,21 @@ use std::fmt;
 
 #[derive(Debug)]
 pub struct CompileError {
+    pub statement: Option<String>,
     pub error: CompileErrorType,
     pub location: Location,
 }
 
+impl CompileError {
+    pub fn update_statement_info(&mut self, statement: String) {
+        self.statement = Some(statement);
+    }
+}
+
 impl From<ParseError> for CompileError {
     fn from(error: ParseError) -> Self {
         CompileError {
+            statement: None,
             error: CompileErrorType::Parse(error.error),
             location: error.location,
         }
@@ -70,21 +78,31 @@ impl CompileError {
 
 impl fmt::Display for CompileError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match &self.error {
-            CompileErrorType::Assign(target) => write!(f, "can't assign to {}", target),
-            CompileErrorType::Delete(target) => write!(f, "can't delete {}", target),
-            CompileErrorType::ExpectExpr => write!(f, "Expecting expression, got statement"),
-            CompileErrorType::Parse(err) => write!(f, "{}", err),
-            CompileErrorType::SyntaxError(err) => write!(f, "{}", err),
-            CompileErrorType::StarArgs => write!(f, "Two starred expressions in assignment"),
-            CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"),
-            CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"),
-            CompileErrorType::InvalidReturn => write!(f, "'return' outside function"),
-            CompileErrorType::InvalidYield => write!(f, "'yield' outside function"),
-        }?;
+        let error_desc = match &self.error {
+            CompileErrorType::Assign(target) => format!("can't assign to {}", target),
+            CompileErrorType::Delete(target) => format!("can't delete {}", target),
+            CompileErrorType::ExpectExpr => "Expecting expression, got statement".to_string(),
+            CompileErrorType::Parse(err) => err.to_string(),
+            CompileErrorType::SyntaxError(err) => err.to_string(),
+            CompileErrorType::StarArgs => "Two starred expressions in assignment".to_string(),
+            CompileErrorType::InvalidBreak => "'break' outside loop".to_string(),
+            CompileErrorType::InvalidContinue => "'continue' outside loop".to_string(),
+            CompileErrorType::InvalidReturn => "'return' outside function".to_string(),
+            CompileErrorType::InvalidYield => "'yield' outside function".to_string(),
+        };
 
-        // Print line number:
-        write!(f, " at {}", self.location)
+        if self.statement.is_some() && self.location.column() > 0 {
+            // visualize the error, when location and statement are provided
+            write!(
+                f,
+                "\n{}\n{}",
+                self.statement.clone().unwrap(),
+                self.location.visualize(&error_desc)
+            )
+        } else {
+            // print line number
+            write!(f, "{} at {}", error_desc, self.location)
+        }
     }
 }
 
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 61a76ddf..217265b4 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -142,6 +142,7 @@ pub struct SymbolTableError {
 impl From<SymbolTableError> for CompileError {
     fn from(error: SymbolTableError) -> Self {
         CompileError {
+            statement: None,
             error: CompileErrorType::SyntaxError(error.error),
             location: error.location,
         }

From aff1a6d53cef97593301b6c7057bf60717938913 Mon Sep 17 00:00:00 2001
From: Aviv Palivoda <palaviv@gmail.com>
Date: Fri, 29 Nov 2019 11:32:45 +0200
Subject: [PATCH 095/459] Fix caret diagnostics for mutiple lines

---
 src/error.rs | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/error.rs b/src/error.rs
index 346ff3ac..f4269376 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -91,18 +91,17 @@ impl fmt::Display for CompileError {
             CompileErrorType::InvalidYield => "'yield' outside function".to_string(),
         };
 
-        if self.statement.is_some() && self.location.column() > 0 {
-            // visualize the error, when location and statement are provided
-            write!(
-                f,
-                "\n{}\n{}",
-                self.statement.clone().unwrap(),
-                self.location.visualize(&error_desc)
-            )
-        } else {
-            // print line number
-            write!(f, "{} at {}", error_desc, self.location)
+        if let Some(statement) = &self.statement {
+            if self.location.column() > 0 {
+                if let Some(line) = statement.lines().nth(self.location.row() - 1) {
+                    // visualize the error, when location and statement are provided
+                    return write!(f, "\n{}\n{}", line, self.location.visualize(&error_desc));
+                }
+            }
         }
+
+        // print line number
+        write!(f, "{} at {}", error_desc, self.location)
     }
 }
 

From 23993c2833ccdf554ac82030f2f07b3ec1cb3ce4 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Sat, 14 Dec 2019 22:05:15 +0900
Subject: [PATCH 096/459] implement SyntaxError attributes

---
 src/compile.rs     | 19 ++++++++++++++++---
 src/error.rs       |  7 +++++++
 src/symboltable.rs |  1 +
 3 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index e4bf2084..39f550d0 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -63,17 +63,21 @@ pub fn compile(
     match mode {
         Mode::Exec => {
             let ast = parser::parse_program(source)?;
-            compile_program(ast, source_path, optimize)
+            compile_program(ast, source_path.clone(), optimize)
         }
         Mode::Eval => {
             let statement = parser::parse_statement(source)?;
-            compile_statement_eval(statement, source_path, optimize)
+            compile_statement_eval(statement, source_path.clone(), optimize)
         }
         Mode::Single => {
             let ast = parser::parse_program(source)?;
-            compile_program_single(ast, source_path, optimize)
+            compile_program_single(ast, source_path.clone(), optimize)
         }
     }
+    .map_err(|mut err| {
+        err.update_source_path(&source_path);
+        err
+    })
 }
 
 /// A helper function for the shared code of the different compile functions
@@ -258,6 +262,7 @@ impl<O: OutputStream> Compiler<O> {
                     statement: None,
                     error: CompileErrorType::ExpectExpr,
                     location: statement.location.clone(),
+                    source_path: None,
                 });
             }
         }
@@ -537,6 +542,7 @@ impl<O: OutputStream> Compiler<O> {
                         statement: None,
                         error: CompileErrorType::InvalidBreak,
                         location: statement.location.clone(),
+                        source_path: None,
                     });
                 }
                 self.emit(Instruction::Break);
@@ -547,6 +553,7 @@ impl<O: OutputStream> Compiler<O> {
                         statement: None,
                         error: CompileErrorType::InvalidContinue,
                         location: statement.location.clone(),
+                        source_path: None,
                     });
                 }
                 self.emit(Instruction::Continue);
@@ -557,6 +564,7 @@ impl<O: OutputStream> Compiler<O> {
                         statement: None,
                         error: CompileErrorType::InvalidReturn,
                         location: statement.location.clone(),
+                        source_path: None,
                     });
                 }
                 match value {
@@ -635,6 +643,7 @@ impl<O: OutputStream> Compiler<O> {
                     statement: None,
                     error: CompileErrorType::Delete(expression.name()),
                     location: self.current_source_location.clone(),
+                    source_path: None,
                 });
             }
         }
@@ -1339,6 +1348,7 @@ impl<O: OutputStream> Compiler<O> {
                                 statement: None,
                                 error: CompileErrorType::StarArgs,
                                 location: self.current_source_location.clone(),
+                                source_path: None,
                             });
                         } else {
                             seen_star = true;
@@ -1369,6 +1379,7 @@ impl<O: OutputStream> Compiler<O> {
                     statement: None,
                     error: CompileErrorType::Assign(target.name()),
                     location: self.current_source_location.clone(),
+                    source_path: None,
                 });
             }
         }
@@ -1654,6 +1665,7 @@ impl<O: OutputStream> Compiler<O> {
                         statement: Option::None,
                         error: CompileErrorType::InvalidYield,
                         location: self.current_source_location.clone(),
+                        source_path: Option::None,
                     });
                 }
                 self.mark_generator();
@@ -1751,6 +1763,7 @@ impl<O: OutputStream> Compiler<O> {
                         "Invalid starred expression",
                     )),
                     location: self.current_source_location.clone(),
+                    source_path: Option::None,
                 });
             }
             IfExpression { test, body, orelse } => {
diff --git a/src/error.rs b/src/error.rs
index f4269376..ae53a003 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -10,12 +10,18 @@ pub struct CompileError {
     pub statement: Option<String>,
     pub error: CompileErrorType,
     pub location: Location,
+    pub source_path: Option<String>,
 }
 
 impl CompileError {
     pub fn update_statement_info(&mut self, statement: String) {
         self.statement = Some(statement);
     }
+
+    pub fn update_source_path(&mut self, source_path: &str) {
+        debug_assert!(self.source_path.is_none());
+        self.source_path = Some(source_path.to_owned());
+    }
 }
 
 impl From<ParseError> for CompileError {
@@ -24,6 +30,7 @@ impl From<ParseError> for CompileError {
             statement: None,
             error: CompileErrorType::Parse(error.error),
             location: error.location,
+            source_path: None,
         }
     }
 }
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 217265b4..28138546 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -145,6 +145,7 @@ impl From<SymbolTableError> for CompileError {
             statement: None,
             error: CompileErrorType::SyntaxError(error.error),
             location: error.location,
+            source_path: None,
         }
     }
 }

From 8adb8b5105b5d6304beafe1cbbd782db332f4039 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Fri, 20 Dec 2019 22:39:01 +0900
Subject: [PATCH 097/459] Fix clippy warnings

---
 src/compile.rs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 39f550d0..24df923b 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1083,9 +1083,7 @@ impl<O: OutputStream> Compiler<O> {
         // Doc string value:
         self.emit(Instruction::LoadConst {
             value: match doc_str {
-                Some(doc) => bytecode::Constant::String {
-                    value: doc.to_string(),
-                },
+                Some(doc) => bytecode::Constant::String { value: doc },
                 None => bytecode::Constant::None, // set docstring None if not declared
             },
         });
@@ -2199,7 +2197,7 @@ fn get_doc(body: &[ast::Statement]) -> (&[ast::Statement], Option<String>) {
         if let ast::StatementType::Expression { ref expression } = val.node {
             if let ast::ExpressionType::String { value } = &expression.node {
                 if let Some(value) = try_get_constant_string(value) {
-                    return (body_rest, Some(value.to_string()));
+                    return (body_rest, Some(value));
                 }
             }
         }

From de7f9efd5403d42139c143fd6664f3b2c313b4c7 Mon Sep 17 00:00:00 2001
From: Aviv Palivoda <palaviv@gmail.com>
Date: Fri, 13 Dec 2019 11:33:19 +0200
Subject: [PATCH 098/459] Support expression in f-strings spec

---
 src/compile.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index 24df923b..d02d3d38 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -2108,10 +2108,17 @@ impl<O: OutputStream> Compiler<O> {
                     conversion,
                     spec,
                 } => {
+                    match spec {
+                        Some(spec) => self.compile_string(spec)?,
+                        None => self.emit(Instruction::LoadConst {
+                            value: bytecode::Constant::String {
+                                value: String::new(),
+                            },
+                        }),
+                    };
                     self.compile_expression(value)?;
                     self.emit(Instruction::FormatValue {
                         conversion: conversion.map(compile_conversion_flag),
-                        spec: spec.clone(),
                     });
                 }
             }

From 3827dbfc49fc2549ce34ea16f997277c1a165392 Mon Sep 17 00:00:00 2001
From: Aviv Palivoda <palaviv@gmail.com>
Date: Sat, 21 Dec 2019 11:27:52 +0200
Subject: [PATCH 099/459] Parse FormattedValue spec in symboltable

---
 src/symboltable.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 28138546..955ee48c 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -726,8 +726,15 @@ impl SymbolTableBuilder {
     fn scan_string_group(&mut self, group: &ast::StringGroup) -> SymbolTableResult {
         match group {
             ast::StringGroup::Constant { .. } => {}
-            ast::StringGroup::FormattedValue { value, .. } => {
+            ast::StringGroup::FormattedValue {
+                value,
+                conversion: _,
+                spec,
+            } => {
                 self.scan_expression(value, &ExpressionContext::Load)?;
+                if let Some(spec) = spec {
+                    self.scan_string_group(spec)?;
+                }
             }
             ast::StringGroup::Joined { values } => {
                 for subgroup in values {

From 9a599e81a4be644f9395219863a020c9d2c067d4 Mon Sep 17 00:00:00 2001
From: Aviv Palivoda <palaviv@gmail.com>
Date: Fri, 27 Dec 2019 10:27:51 +0200
Subject: [PATCH 100/459] Fix clippy warning

---
 src/symboltable.rs | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 955ee48c..c3e2099b 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -726,11 +726,7 @@ impl SymbolTableBuilder {
     fn scan_string_group(&mut self, group: &ast::StringGroup) -> SymbolTableResult {
         match group {
             ast::StringGroup::Constant { .. } => {}
-            ast::StringGroup::FormattedValue {
-                value,
-                conversion: _,
-                spec,
-            } => {
+            ast::StringGroup::FormattedValue { value, spec, .. } => {
                 self.scan_expression(value, &ExpressionContext::Load)?;
                 if let Some(spec) = spec {
                     self.scan_string_group(spec)?;

From 7860fc877d2c7ad85f5122aa685e9829ea8490ca Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 5 Feb 2020 19:15:18 +0900
Subject: [PATCH 101/459] &str::to_string -> &str::to_owned for literals

---
 src/compile.rs | 22 +++++++++++-----------
 src/error.rs   | 14 +++++++-------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index d02d3d38..8ea3bc52 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -88,7 +88,7 @@ fn with_compiler(
 ) -> Result<CodeObject, CompileError> {
     let mut compiler = Compiler::new(optimize);
     compiler.source_path = Some(source_path);
-    compiler.push_new_code_object("<module>".to_string());
+    compiler.push_new_code_object("<module>".to_owned());
     f(&mut compiler)?;
     let code = compiler.pop_code_object();
     trace!("Compilation completed: {:?}", code);
@@ -897,7 +897,7 @@ impl<O: OutputStream> Compiler<O> {
             // key:
             self.emit(Instruction::LoadConst {
                 value: bytecode::Constant::String {
-                    value: "return".to_string(),
+                    value: "return".to_owned(),
                 },
             });
             // value:
@@ -989,11 +989,11 @@ impl<O: OutputStream> Compiler<O> {
         let (new_body, doc_str) = get_doc(body);
 
         self.emit(Instruction::LoadName {
-            name: "__name__".to_string(),
+            name: "__name__".to_owned(),
             scope: bytecode::NameScope::Global,
         });
         self.emit(Instruction::StoreName {
-            name: "__module__".to_string(),
+            name: "__module__".to_owned(),
             scope: bytecode::NameScope::Free,
         });
         self.emit(Instruction::LoadConst {
@@ -1002,7 +1002,7 @@ impl<O: OutputStream> Compiler<O> {
             },
         });
         self.emit(Instruction::StoreName {
-            name: "__qualname__".to_string(),
+            name: "__qualname__".to_owned(),
             scope: bytecode::NameScope::Free,
         });
         self.compile_statements(new_body)?;
@@ -1090,7 +1090,7 @@ impl<O: OutputStream> Compiler<O> {
 
         self.emit(Instruction::Rotate { amount: 2 });
         self.emit(Instruction::StoreAttr {
-            name: "__doc__".to_string(),
+            name: "__doc__".to_owned(),
         });
     }
 
@@ -1171,7 +1171,7 @@ impl<O: OutputStream> Compiler<O> {
             self.set_label(check_asynciter_label);
             self.emit(Instruction::Duplicate);
             self.emit(Instruction::LoadName {
-                name: "StopAsyncIteration".to_string(),
+                name: "StopAsyncIteration".to_owned(),
                 scope: bytecode::NameScope::Global,
             });
             self.emit(Instruction::CompareOperation {
@@ -1732,7 +1732,7 @@ impl<O: OutputStream> Compiler<O> {
                     func: FunctionContext::Function,
                 };
 
-                let name = "<lambda>".to_string();
+                let name = "<lambda>".to_owned();
                 self.enter_function(&name, args)?;
                 self.compile_expression(body)?;
                 self.emit(Instruction::ReturnValue);
@@ -1933,7 +1933,7 @@ impl<O: OutputStream> Compiler<O> {
         // Create magnificent function <listcomp>:
         self.push_output(CodeObject::new(
             Default::default(),
-            vec![".0".to_string()],
+            vec![".0".to_owned()],
             Varargs::None,
             vec![],
             Varargs::None,
@@ -2264,8 +2264,8 @@ mod tests {
 
     fn compile_exec(source: &str) -> CodeObject {
         let mut compiler: Compiler = Default::default();
-        compiler.source_path = Some("source_path".to_string());
-        compiler.push_new_code_object("<module>".to_string());
+        compiler.source_path = Some("source_path".to_owned());
+        compiler.push_new_code_object("<module>".to_owned());
         let ast = parser::parse_program(&source.to_string()).unwrap();
         let symbol_scope = make_symbol_table(&ast).unwrap();
         compiler.compile_program(&ast, symbol_scope).unwrap();
diff --git a/src/error.rs b/src/error.rs
index ae53a003..836e479a 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -62,7 +62,7 @@ impl CompileError {
             match parse {
                 ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true,
                 ParseErrorType::UnrecognizedToken(token, expected) => {
-                    *token == Tok::Indent || expected.clone() == Some("Indent".to_string())
+                    *token == Tok::Indent || expected.clone() == Some("Indent".to_owned())
                 }
                 _ => false,
             }
@@ -88,14 +88,14 @@ impl fmt::Display for CompileError {
         let error_desc = match &self.error {
             CompileErrorType::Assign(target) => format!("can't assign to {}", target),
             CompileErrorType::Delete(target) => format!("can't delete {}", target),
-            CompileErrorType::ExpectExpr => "Expecting expression, got statement".to_string(),
+            CompileErrorType::ExpectExpr => "Expecting expression, got statement".to_owned(),
             CompileErrorType::Parse(err) => err.to_string(),
             CompileErrorType::SyntaxError(err) => err.to_string(),
-            CompileErrorType::StarArgs => "Two starred expressions in assignment".to_string(),
-            CompileErrorType::InvalidBreak => "'break' outside loop".to_string(),
-            CompileErrorType::InvalidContinue => "'continue' outside loop".to_string(),
-            CompileErrorType::InvalidReturn => "'return' outside function".to_string(),
-            CompileErrorType::InvalidYield => "'yield' outside function".to_string(),
+            CompileErrorType::StarArgs => "Two starred expressions in assignment".to_owned(),
+            CompileErrorType::InvalidBreak => "'break' outside loop".to_owned(),
+            CompileErrorType::InvalidContinue => "'continue' outside loop".to_owned(),
+            CompileErrorType::InvalidReturn => "'return' outside function".to_owned(),
+            CompileErrorType::InvalidYield => "'yield' outside function".to_owned(),
         };
 
         if let Some(statement) = &self.statement {

From 04c29f384e09f356b47cd1ccd552edc65a665158 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 5 Feb 2020 22:03:36 +0900
Subject: [PATCH 102/459] &str::to_string -> &str::to_owned for variables

---
 src/compile.rs     | 36 ++++++++++++++++++------------------
 src/symboltable.rs |  6 +++---
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 8ea3bc52..dbe938fd 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -290,7 +290,7 @@ impl<O: OutputStream> Compiler<O> {
     fn load_name(&mut self, name: &str) {
         let scope = self.scope_for_name(name);
         self.emit(Instruction::LoadName {
-            name: name.to_string(),
+            name: name.to_owned(),
             scope,
         });
     }
@@ -298,7 +298,7 @@ impl<O: OutputStream> Compiler<O> {
     fn store_name(&mut self, name: &str) {
         let scope = self.scope_for_name(name);
         self.emit(Instruction::StoreName {
-            name: name.to_string(),
+            name: name.to_owned(),
             scope,
         });
     }
@@ -359,7 +359,7 @@ impl<O: OutputStream> Compiler<O> {
                     for name in names {
                         // import symbol from module:
                         self.emit(Instruction::ImportFrom {
-                            name: name.symbol.to_string(),
+                            name: name.symbol.to_owned(),
                         });
 
                         // Store module under proper name:
@@ -619,13 +619,13 @@ impl<O: OutputStream> Compiler<O> {
         match &expression.node {
             ast::ExpressionType::Identifier { name } => {
                 self.emit(Instruction::DeleteName {
-                    name: name.to_string(),
+                    name: name.to_owned(),
                 });
             }
             ast::ExpressionType::Attribute { value, name } => {
                 self.compile_expression(value)?;
                 self.emit(Instruction::DeleteAttr {
-                    name: name.to_string(),
+                    name: name.to_owned(),
                 });
             }
             ast::ExpressionType::Subscript { a, b } => {
@@ -701,7 +701,7 @@ impl<O: OutputStream> Compiler<O> {
             compile_varargs(&args.kwarg),
             self.source_path.clone().unwrap(),
             line_number,
-            name.to_string(),
+            name.to_owned(),
         ));
         self.enter_scope();
 
@@ -909,7 +909,7 @@ impl<O: OutputStream> Compiler<O> {
             if let Some(annotation) = &arg.annotation {
                 self.emit(Instruction::LoadConst {
                     value: bytecode::Constant::String {
-                        value: arg.arg.to_string(),
+                        value: arg.arg.to_owned(),
                     },
                 });
                 self.compile_expression(&annotation)?;
@@ -982,7 +982,7 @@ impl<O: OutputStream> Compiler<O> {
             Varargs::None,
             self.source_path.clone().unwrap(),
             line_number,
-            name.to_string(),
+            name.to_owned(),
         ));
         self.enter_scope();
 
@@ -1022,7 +1022,7 @@ impl<O: OutputStream> Compiler<O> {
         });
         self.emit(Instruction::LoadConst {
             value: bytecode::Constant::String {
-                value: name.to_string(),
+                value: name.to_owned(),
             },
         });
 
@@ -1044,7 +1044,7 @@ impl<O: OutputStream> Compiler<O> {
             for keyword in keywords {
                 if let Some(name) = &keyword.name {
                     kwarg_names.push(bytecode::Constant::String {
-                        value: name.to_string(),
+                        value: name.to_owned(),
                     });
                 } else {
                     // This means **kwargs!
@@ -1308,7 +1308,7 @@ impl<O: OutputStream> Compiler<O> {
             });
             self.emit(Instruction::LoadConst {
                 value: bytecode::Constant::String {
-                    value: name.to_string(),
+                    value: name.to_owned(),
                 },
             });
             self.emit(Instruction::StoreSubscript);
@@ -1332,7 +1332,7 @@ impl<O: OutputStream> Compiler<O> {
             ast::ExpressionType::Attribute { value, name } => {
                 self.compile_expression(value)?;
                 self.emit(Instruction::StoreAttr {
-                    name: name.to_string(),
+                    name: name.to_owned(),
                 });
             }
             ast::ExpressionType::List { elements } | ast::ExpressionType::Tuple { elements } => {
@@ -1605,7 +1605,7 @@ impl<O: OutputStream> Compiler<O> {
             Attribute { value, name } => {
                 self.compile_expression(value)?;
                 self.emit(Instruction::LoadAttr {
-                    name: name.to_string(),
+                    name: name.to_owned(),
                 });
             }
             Compare { vals, ops } => {
@@ -1795,7 +1795,7 @@ impl<O: OutputStream> Compiler<O> {
                     if let Some(name) = &keyword.name {
                         self.emit(Instruction::LoadConst {
                             value: bytecode::Constant::String {
-                                value: name.to_string(),
+                                value: name.to_owned(),
                             },
                         });
                         self.compile_expression(&keyword.value)?;
@@ -1858,7 +1858,7 @@ impl<O: OutputStream> Compiler<O> {
                 for keyword in keywords {
                     if let Some(name) = &keyword.name {
                         kwarg_names.push(bytecode::Constant::String {
-                            value: name.to_string(),
+                            value: name.to_owned(),
                         });
                     } else {
                         // This means **kwargs!
@@ -1927,7 +1927,7 @@ impl<O: OutputStream> Compiler<O> {
             ast::ComprehensionKind::Set { .. } => "<setcomp>",
             ast::ComprehensionKind::Dict { .. } => "<dictcomp>",
         }
-        .to_string();
+        .to_owned();
 
         let line_number = self.get_source_line_number();
         // Create magnificent function <listcomp>:
@@ -2099,7 +2099,7 @@ impl<O: OutputStream> Compiler<O> {
                 ast::StringGroup::Constant { value } => {
                     self.emit(Instruction::LoadConst {
                         value: bytecode::Constant::String {
-                            value: value.to_string(),
+                            value: value.to_owned(),
                         },
                     });
                 }
@@ -2266,7 +2266,7 @@ mod tests {
         let mut compiler: Compiler = Default::default();
         compiler.source_path = Some("source_path".to_owned());
         compiler.push_new_code_object("<module>".to_owned());
-        let ast = parser::parse_program(&source.to_string()).unwrap();
+        let ast = parser::parse_program(source).unwrap();
         let symbol_scope = make_symbol_table(&ast).unwrap();
         compiler.compile_program(&ast, symbol_scope).unwrap();
         compiler.pop_code_object()
diff --git a/src/symboltable.rs b/src/symboltable.rs
index c3e2099b..d08aa4f2 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -105,7 +105,7 @@ pub struct Symbol {
 impl Symbol {
     fn new(name: &str) -> Self {
         Symbol {
-            name: name.to_string(),
+            name: name.to_owned(),
             // table,
             scope: SymbolScope::Unknown,
             is_param: false,
@@ -304,7 +304,7 @@ impl SymbolTableBuilder {
     }
 
     fn enter_scope(&mut self, name: &str, typ: SymbolTableType, line_number: usize) {
-        let table = SymbolTable::new(name.to_string(), typ, line_number);
+        let table = SymbolTable::new(name.to_owned(), typ, line_number);
         self.tables.push(table);
     }
 
@@ -793,7 +793,7 @@ impl SymbolTableBuilder {
         // Insert symbol when required:
         if !containing {
             let symbol = Symbol::new(name);
-            table.symbols.insert(name.to_string(), symbol);
+            table.symbols.insert(name.to_owned(), symbol);
         }
 
         // Set proper flags on symbol:

From 867152880a3d9093a1df5a899113d7994156c929 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Sun, 1 Mar 2020 02:22:50 +0900
Subject: [PATCH 103/459] positional only parameter support

---
 src/compile.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/compile.rs b/src/compile.rs
index dbe938fd..b8552518 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -165,6 +165,7 @@ impl<O: OutputStream> Compiler<O> {
         let line_number = self.get_source_line_number();
         self.push_output(CodeObject::new(
             Default::default(),
+            0,
             Vec::new(),
             Varargs::None,
             Vec::new(),
@@ -695,6 +696,7 @@ impl<O: OutputStream> Compiler<O> {
         let line_number = self.get_source_line_number();
         self.push_output(CodeObject::new(
             flags,
+            args.posonlyargs_count,
             args.args.iter().map(|a| a.arg.clone()).collect(),
             compile_varargs(&args.vararg),
             args.kwonlyargs.iter().map(|a| a.arg.clone()).collect(),
@@ -976,6 +978,7 @@ impl<O: OutputStream> Compiler<O> {
         let line_number = self.get_source_line_number();
         self.push_output(CodeObject::new(
             Default::default(),
+            0,
             vec![],
             Varargs::None,
             vec![],
@@ -1933,6 +1936,7 @@ impl<O: OutputStream> Compiler<O> {
         // Create magnificent function <listcomp>:
         self.push_output(CodeObject::new(
             Default::default(),
+            1,
             vec![".0".to_owned()],
             Varargs::None,
             vec![],

From 04606e905730a1d010afaa30af50c6f365a759e7 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 13 Mar 2020 09:01:10 -0500
Subject: [PATCH 104/459] Make Location Copy and remove all location.clone()s

---
 src/compile.rs | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index b8552518..fb1fb33a 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -262,7 +262,7 @@ impl<O: OutputStream> Compiler<O> {
                 return Err(CompileError {
                     statement: None,
                     error: CompileErrorType::ExpectExpr,
-                    location: statement.location.clone(),
+                    location: statement.location,
                     source_path: None,
                 });
             }
@@ -306,7 +306,7 @@ impl<O: OutputStream> Compiler<O> {
 
     fn compile_statement(&mut self, statement: &ast::Statement) -> Result<(), CompileError> {
         trace!("Compiling {:?}", statement);
-        self.set_source_location(&statement.location);
+        self.set_source_location(statement.location);
         use ast::StatementType::*;
 
         match &statement.node {
@@ -542,7 +542,7 @@ impl<O: OutputStream> Compiler<O> {
                     return Err(CompileError {
                         statement: None,
                         error: CompileErrorType::InvalidBreak,
-                        location: statement.location.clone(),
+                        location: statement.location,
                         source_path: None,
                     });
                 }
@@ -553,7 +553,7 @@ impl<O: OutputStream> Compiler<O> {
                     return Err(CompileError {
                         statement: None,
                         error: CompileErrorType::InvalidContinue,
-                        location: statement.location.clone(),
+                        location: statement.location,
                         source_path: None,
                     });
                 }
@@ -564,7 +564,7 @@ impl<O: OutputStream> Compiler<O> {
                     return Err(CompileError {
                         statement: None,
                         error: CompileErrorType::InvalidReturn,
-                        location: statement.location.clone(),
+                        location: statement.location,
                         source_path: None,
                     });
                 }
@@ -643,7 +643,7 @@ impl<O: OutputStream> Compiler<O> {
                 return Err(CompileError {
                     statement: None,
                     error: CompileErrorType::Delete(expression.name()),
-                    location: self.current_source_location.clone(),
+                    location: self.current_source_location,
                     source_path: None,
                 });
             }
@@ -1348,7 +1348,7 @@ impl<O: OutputStream> Compiler<O> {
                             return Err(CompileError {
                                 statement: None,
                                 error: CompileErrorType::StarArgs,
-                                location: self.current_source_location.clone(),
+                                location: self.current_source_location,
                                 source_path: None,
                             });
                         } else {
@@ -1379,7 +1379,7 @@ impl<O: OutputStream> Compiler<O> {
                 return Err(CompileError {
                     statement: None,
                     error: CompileErrorType::Assign(target.name()),
-                    location: self.current_source_location.clone(),
+                    location: self.current_source_location,
                     source_path: None,
                 });
             }
@@ -1570,7 +1570,7 @@ impl<O: OutputStream> Compiler<O> {
 
     fn compile_expression(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
         trace!("Compiling {:?}", expression);
-        self.set_source_location(&expression.location);
+        self.set_source_location(expression.location);
 
         use ast::ExpressionType::*;
         match &expression.node {
@@ -1665,7 +1665,7 @@ impl<O: OutputStream> Compiler<O> {
                     return Err(CompileError {
                         statement: Option::None,
                         error: CompileErrorType::InvalidYield,
-                        location: self.current_source_location.clone(),
+                        location: self.current_source_location,
                         source_path: Option::None,
                     });
                 }
@@ -1763,7 +1763,7 @@ impl<O: OutputStream> Compiler<O> {
                     error: CompileErrorType::SyntaxError(std::string::String::from(
                         "Invalid starred expression",
                     )),
-                    location: self.current_source_location.clone(),
+                    location: self.current_source_location,
                     source_path: Option::None,
                 });
             }
@@ -2182,8 +2182,8 @@ impl<O: OutputStream> Compiler<O> {
         self.current_output().set_label(label)
     }
 
-    fn set_source_location(&mut self, location: &ast::Location) {
-        self.current_source_location = location.clone();
+    fn set_source_location(&mut self, location: ast::Location) {
+        self.current_source_location = location;
     }
 
     fn get_source_line_number(&mut self) -> usize {

From 00f9d2247d3df1a02394ec5d98352ccf7ae4ab8d Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 13 Mar 2020 20:57:44 -0500
Subject: [PATCH 105/459] Make bytecode::Location Copy

---
 src/peephole.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/peephole.rs b/src/peephole.rs
index 3a64380b..b8526686 100644
--- a/src/peephole.rs
+++ b/src/peephole.rs
@@ -15,7 +15,7 @@ impl From<Vec<InstructionMetadata>> for InstructionMetadata {
     fn from(metas: Vec<Self>) -> Self {
         debug_assert!(!metas.is_empty(), "`metas` must not be empty");
         InstructionMetadata {
-            loc: metas[0].loc.clone(),
+            loc: metas[0].loc,
             labels: metas
                 .into_iter()
                 .flat_map(|meta| meta.labels.into_iter())

From e9ff68c850c56d7cff4dea47314cc9547836d066 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 13 Mar 2020 21:36:33 -0500
Subject: [PATCH 106/459] Improve compiler error creation

---
 src/compile.rs | 164 +++++++++++++++++++------------------------------
 1 file changed, 63 insertions(+), 101 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index fb1fb33a..034e0f9a 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -19,6 +19,8 @@ use rustpython_parser::{ast, parser};
 
 type BasicOutputStream = PeepholeOptimizer<CodeObjectStream>;
 
+type CompileResult<T> = Result<T, CompileError>;
+
 /// Main structure holding the state of compilation.
 struct Compiler<O: OutputStream = BasicOutputStream> {
     output_stack: Vec<O>,
@@ -59,33 +61,29 @@ pub fn compile(
     mode: Mode,
     source_path: String,
     optimize: u8,
-) -> Result<CodeObject, CompileError> {
+) -> CompileResult<CodeObject> {
     match mode {
         Mode::Exec => {
             let ast = parser::parse_program(source)?;
-            compile_program(ast, source_path.clone(), optimize)
+            compile_program(ast, source_path, optimize)
         }
         Mode::Eval => {
             let statement = parser::parse_statement(source)?;
-            compile_statement_eval(statement, source_path.clone(), optimize)
+            compile_statement_eval(statement, source_path, optimize)
         }
         Mode::Single => {
             let ast = parser::parse_program(source)?;
-            compile_program_single(ast, source_path.clone(), optimize)
+            compile_program_single(ast, source_path, optimize)
         }
     }
-    .map_err(|mut err| {
-        err.update_source_path(&source_path);
-        err
-    })
 }
 
 /// A helper function for the shared code of the different compile functions
 fn with_compiler(
     source_path: String,
     optimize: u8,
-    f: impl FnOnce(&mut Compiler) -> Result<(), CompileError>,
-) -> Result<CodeObject, CompileError> {
+    f: impl FnOnce(&mut Compiler) -> CompileResult<()>,
+) -> CompileResult<CodeObject> {
     let mut compiler = Compiler::new(optimize);
     compiler.source_path = Some(source_path);
     compiler.push_new_code_object("<module>".to_owned());
@@ -100,7 +98,7 @@ pub fn compile_program(
     ast: ast::Program,
     source_path: String,
     optimize: u8,
-) -> Result<CodeObject, CompileError> {
+) -> CompileResult<CodeObject> {
     with_compiler(source_path, optimize, |compiler| {
         let symbol_table = make_symbol_table(&ast)?;
         compiler.compile_program(&ast, symbol_table)
@@ -112,7 +110,7 @@ pub fn compile_statement_eval(
     statement: Vec<ast::Statement>,
     source_path: String,
     optimize: u8,
-) -> Result<CodeObject, CompileError> {
+) -> CompileResult<CodeObject> {
     with_compiler(source_path, optimize, |compiler| {
         let symbol_table = statements_to_symbol_table(&statement)?;
         compiler.compile_statement_eval(&statement, symbol_table)
@@ -124,7 +122,7 @@ pub fn compile_program_single(
     ast: ast::Program,
     source_path: String,
     optimize: u8,
-) -> Result<CodeObject, CompileError> {
+) -> CompileResult<CodeObject> {
     with_compiler(source_path, optimize, |compiler| {
         let symbol_table = make_symbol_table(&ast)?;
         compiler.compile_program_single(&ast, symbol_table)
@@ -157,6 +155,18 @@ impl<O: OutputStream> Compiler<O> {
         }
     }
 
+    fn error(&self, error: CompileErrorType) -> CompileError {
+        self.error_loc(error, self.current_source_location)
+    }
+    fn error_loc(&self, error: CompileErrorType, location: ast::Location) -> CompileError {
+        CompileError {
+            error,
+            location,
+            source_path: self.source_path.clone(),
+            statement: None,
+        }
+    }
+
     fn push_output(&mut self, code: CodeObject) {
         self.output_stack.push(code.into());
     }
@@ -184,7 +194,7 @@ impl<O: OutputStream> Compiler<O> {
         &mut self,
         program: &ast::Program,
         symbol_table: SymbolTable,
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         let size_before = self.output_stack.len();
         self.symbol_table_stack.push(symbol_table);
 
@@ -214,7 +224,7 @@ impl<O: OutputStream> Compiler<O> {
         &mut self,
         program: &ast::Program,
         symbol_table: SymbolTable,
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         self.symbol_table_stack.push(symbol_table);
 
         let mut emitted_return = false;
@@ -253,25 +263,20 @@ impl<O: OutputStream> Compiler<O> {
         &mut self,
         statements: &[ast::Statement],
         symbol_table: SymbolTable,
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         self.symbol_table_stack.push(symbol_table);
         for statement in statements {
             if let ast::StatementType::Expression { ref expression } = statement.node {
                 self.compile_expression(expression)?;
             } else {
-                return Err(CompileError {
-                    statement: None,
-                    error: CompileErrorType::ExpectExpr,
-                    location: statement.location,
-                    source_path: None,
-                });
+                return Err(self.error_loc(CompileErrorType::ExpectExpr, statement.location));
             }
         }
         self.emit(Instruction::ReturnValue);
         Ok(())
     }
 
-    fn compile_statements(&mut self, statements: &[ast::Statement]) -> Result<(), CompileError> {
+    fn compile_statements(&mut self, statements: &[ast::Statement]) -> CompileResult<()> {
         for statement in statements {
             self.compile_statement(statement)?
         }
@@ -304,7 +309,7 @@ impl<O: OutputStream> Compiler<O> {
         });
     }
 
-    fn compile_statement(&mut self, statement: &ast::Statement) -> Result<(), CompileError> {
+    fn compile_statement(&mut self, statement: &ast::Statement) -> CompileResult<()> {
         trace!("Compiling {:?}", statement);
         self.set_source_location(statement.location);
         use ast::StatementType::*;
@@ -443,7 +448,7 @@ impl<O: OutputStream> Compiler<O> {
                         }
                         Ok(end_label)
                     })
-                    .collect::<Result<Vec<_>, CompileError>>()?;
+                    .collect::<CompileResult<Vec<_>>>()?;
 
                 self.compile_statements(body)?;
 
@@ -539,34 +544,21 @@ impl<O: OutputStream> Compiler<O> {
             }
             Break => {
                 if !self.ctx.in_loop {
-                    return Err(CompileError {
-                        statement: None,
-                        error: CompileErrorType::InvalidBreak,
-                        location: statement.location,
-                        source_path: None,
-                    });
+                    return Err(self.error_loc(CompileErrorType::InvalidBreak, statement.location));
                 }
                 self.emit(Instruction::Break);
             }
             Continue => {
                 if !self.ctx.in_loop {
-                    return Err(CompileError {
-                        statement: None,
-                        error: CompileErrorType::InvalidContinue,
-                        location: statement.location,
-                        source_path: None,
-                    });
+                    return Err(
+                        self.error_loc(CompileErrorType::InvalidContinue, statement.location)
+                    );
                 }
                 self.emit(Instruction::Continue);
             }
             Return { value } => {
                 if !self.ctx.in_func() {
-                    return Err(CompileError {
-                        statement: None,
-                        error: CompileErrorType::InvalidReturn,
-                        location: statement.location,
-                        source_path: None,
-                    });
+                    return Err(self.error_loc(CompileErrorType::InvalidReturn, statement.location));
                 }
                 match value {
                     Some(v) => {
@@ -616,7 +608,7 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
-    fn compile_delete(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
+    fn compile_delete(&mut self, expression: &ast::Expression) -> CompileResult<()> {
         match &expression.node {
             ast::ExpressionType::Identifier { name } => {
                 self.emit(Instruction::DeleteName {
@@ -639,19 +631,12 @@ impl<O: OutputStream> Compiler<O> {
                     self.compile_delete(element)?;
                 }
             }
-            _ => {
-                return Err(CompileError {
-                    statement: None,
-                    error: CompileErrorType::Delete(expression.name()),
-                    location: self.current_source_location,
-                    source_path: None,
-                });
-            }
+            _ => return Err(self.error(CompileErrorType::Delete(expression.name()))),
         }
         Ok(())
     }
 
-    fn enter_function(&mut self, name: &str, args: &ast::Parameters) -> Result<(), CompileError> {
+    fn enter_function(&mut self, name: &str, args: &ast::Parameters) -> CompileResult<()> {
         let have_defaults = !args.defaults.is_empty();
         if have_defaults {
             // Construct a tuple:
@@ -710,10 +695,7 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
-    fn prepare_decorators(
-        &mut self,
-        decorator_list: &[ast::Expression],
-    ) -> Result<(), CompileError> {
+    fn prepare_decorators(&mut self, decorator_list: &[ast::Expression]) -> CompileResult<()> {
         for decorator in decorator_list {
             self.compile_expression(decorator)?;
         }
@@ -735,7 +717,7 @@ impl<O: OutputStream> Compiler<O> {
         handlers: &[ast::ExceptHandler],
         orelse: &Option<ast::Suite>,
         finalbody: &Option<ast::Suite>,
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         let mut handler_label = self.new_label();
         let finally_handler_label = self.new_label();
         let else_label = self.new_label();
@@ -849,7 +831,7 @@ impl<O: OutputStream> Compiler<O> {
         decorator_list: &[ast::Expression],
         returns: &Option<ast::Expression>, // TODO: use type hint somehow..
         is_async: bool,
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         // Create bytecode for this function:
         // remember to restore self.ctx.in_loop to the original after the function is compiled
         let prev_ctx = self.ctx;
@@ -962,7 +944,7 @@ impl<O: OutputStream> Compiler<O> {
         bases: &[ast::Expression],
         keywords: &[ast::Keyword],
         decorator_list: &[ast::Expression],
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         let prev_ctx = self.ctx;
         self.ctx = CompileContext {
             func: FunctionContext::NoFunction,
@@ -1102,7 +1084,7 @@ impl<O: OutputStream> Compiler<O> {
         test: &ast::Expression,
         body: &[ast::Statement],
         orelse: &Option<Vec<ast::Statement>>,
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         let start_label = self.new_label();
         let else_label = self.new_label();
         let end_label = self.new_label();
@@ -1138,7 +1120,7 @@ impl<O: OutputStream> Compiler<O> {
         body: &[ast::Statement],
         orelse: &Option<Vec<ast::Statement>>,
         is_async: bool,
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         // Start loop
         let start_label = self.new_label();
         let else_label = self.new_label();
@@ -1223,7 +1205,7 @@ impl<O: OutputStream> Compiler<O> {
         &mut self,
         vals: &[ast::Expression],
         ops: &[ast::Comparison],
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         assert!(!ops.is_empty());
         assert_eq!(vals.len(), ops.len() + 1);
 
@@ -1294,7 +1276,7 @@ impl<O: OutputStream> Compiler<O> {
         target: &ast::Expression,
         annotation: &ast::Expression,
         value: &Option<ast::Expression>,
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         if let Some(value) = value {
             self.compile_expression(value)?;
             self.compile_store(target)?;
@@ -1322,7 +1304,7 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
-    fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> {
+    fn compile_store(&mut self, target: &ast::Expression) -> CompileResult<()> {
         match &target.node {
             ast::ExpressionType::Identifier { name } => {
                 self.store_name(name);
@@ -1345,12 +1327,7 @@ impl<O: OutputStream> Compiler<O> {
                 for (i, element) in elements.iter().enumerate() {
                     if let ast::ExpressionType::Starred { .. } = &element.node {
                         if seen_star {
-                            return Err(CompileError {
-                                statement: None,
-                                error: CompileErrorType::StarArgs,
-                                location: self.current_source_location,
-                                source_path: None,
-                            });
+                            return Err(self.error(CompileErrorType::StarArgs));
                         } else {
                             seen_star = true;
                             self.emit(Instruction::UnpackEx {
@@ -1375,14 +1352,7 @@ impl<O: OutputStream> Compiler<O> {
                     }
                 }
             }
-            _ => {
-                return Err(CompileError {
-                    statement: None,
-                    error: CompileErrorType::Assign(target.name()),
-                    location: self.current_source_location,
-                    source_path: None,
-                });
-            }
+            _ => return Err(self.error(CompileErrorType::Assign(target.name()))),
         }
 
         Ok(())
@@ -1420,7 +1390,7 @@ impl<O: OutputStream> Compiler<O> {
         expression: &ast::Expression,
         condition: bool,
         target_label: Label,
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         // Compile expression for test, and jump to label if false
         match &expression.node {
             ast::ExpressionType::BoolOp { op, values } => {
@@ -1498,7 +1468,7 @@ impl<O: OutputStream> Compiler<O> {
         &mut self,
         op: &ast::BooleanOperator,
         values: &[ast::Expression],
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         let end_label = self.new_label();
 
         let (last_value, values) = values.split_last().unwrap();
@@ -1524,7 +1494,7 @@ impl<O: OutputStream> Compiler<O> {
     fn compile_dict(
         &mut self,
         pairs: &[(Option<ast::Expression>, ast::Expression)],
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         let mut size = 0;
         let mut has_unpacking = false;
         for (is_unpacking, subpairs) in &pairs.iter().group_by(|e| e.0.is_none()) {
@@ -1568,7 +1538,7 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
-    fn compile_expression(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
+    fn compile_expression(&mut self, expression: &ast::Expression) -> CompileResult<()> {
         trace!("Compiling {:?}", expression);
         self.set_source_location(expression.location);
 
@@ -1662,12 +1632,7 @@ impl<O: OutputStream> Compiler<O> {
             }
             Yield { value } => {
                 if !self.ctx.in_func() {
-                    return Err(CompileError {
-                        statement: Option::None,
-                        error: CompileErrorType::InvalidYield,
-                        location: self.current_source_location,
-                        source_path: Option::None,
-                    });
+                    return Err(self.error(CompileErrorType::InvalidYield));
                 }
                 self.mark_generator();
                 match value {
@@ -1758,14 +1723,11 @@ impl<O: OutputStream> Compiler<O> {
                 self.compile_comprehension(kind, generators)?;
             }
             Starred { .. } => {
-                return Err(CompileError {
-                    statement: Option::None,
-                    error: CompileErrorType::SyntaxError(std::string::String::from(
+                return Err(
+                    self.error(CompileErrorType::SyntaxError(std::string::String::from(
                         "Invalid starred expression",
-                    )),
-                    location: self.current_source_location,
-                    source_path: Option::None,
-                });
+                    ))),
+                );
             }
             IfExpression { test, body, orelse } => {
                 let no_label = self.new_label();
@@ -1784,7 +1746,7 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
-    fn compile_keywords(&mut self, keywords: &[ast::Keyword]) -> Result<(), CompileError> {
+    fn compile_keywords(&mut self, keywords: &[ast::Keyword]) -> CompileResult<()> {
         let mut size = 0;
         for (is_unpacking, subkeywords) in &keywords.iter().group_by(|e| e.name.is_none()) {
             if is_unpacking {
@@ -1828,7 +1790,7 @@ impl<O: OutputStream> Compiler<O> {
         function: &ast::Expression,
         args: &[ast::Expression],
         keywords: &[ast::Keyword],
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         self.compile_expression(function)?;
         let count = args.len() + keywords.len();
 
@@ -1889,7 +1851,7 @@ impl<O: OutputStream> Compiler<O> {
 
     // Given a vector of expr / star expr generate code which gives either
     // a list of expressions on the stack, or a list of tuples.
-    fn gather_elements(&mut self, elements: &[ast::Expression]) -> Result<bool, CompileError> {
+    fn gather_elements(&mut self, elements: &[ast::Expression]) -> CompileResult<bool> {
         // First determine if we have starred elements:
         let has_stars = elements.iter().any(|e| {
             if let ast::ExpressionType::Starred { .. } = &e.node {
@@ -1920,7 +1882,7 @@ impl<O: OutputStream> Compiler<O> {
         &mut self,
         kind: &ast::ComprehensionKind,
         generators: &[ast::Comprehension],
-    ) -> Result<(), CompileError> {
+    ) -> CompileResult<()> {
         // We must have at least one generator:
         assert!(!generators.is_empty());
 
@@ -2087,7 +2049,7 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
-    fn compile_string(&mut self, string: &ast::StringGroup) -> Result<(), CompileError> {
+    fn compile_string(&mut self, string: &ast::StringGroup) -> CompileResult<()> {
         if let Some(value) = try_get_constant_string(string) {
             self.emit(Instruction::LoadConst {
                 value: bytecode::Constant::String { value },

From d5b9e6b93a74e14867f8e9976947d04a915ca57c Mon Sep 17 00:00:00 2001
From: coolreader18 <33094578+coolreader18@users.noreply.github.com>
Date: Sun, 5 Jan 2020 00:17:45 -0600
Subject: [PATCH 107/459] Partially implement async generators

---
 src/compile.rs | 57 +++++++++++++++++++++++++++++++++++++++++++++++---
 src/error.rs   |  6 ++++++
 2 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 034e0f9a..f4d088c4 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -39,7 +39,7 @@ struct CompileContext {
     func: FunctionContext,
 }
 
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, PartialEq)]
 enum FunctionContext {
     NoFunction,
     Function,
@@ -562,6 +562,17 @@ impl<O: OutputStream> Compiler<O> {
                 }
                 match value {
                     Some(v) => {
+<<<<<<< Updated upstream
+=======
+                        if self.ctx.func == FunctionContext::AsyncFunction
+                            && self.current_output().is_generator()
+                        {
+                            return Err(self.error_loc(
+                                CompileErrorType::AsyncReturnValue,
+                                statement.location.clone(),
+                            ));
+                        }
+>>>>>>> Stashed changes
                         self.compile_expression(v)?;
                     }
                     None => {
@@ -1543,6 +1554,8 @@ impl<O: OutputStream> Compiler<O> {
         self.set_source_location(expression.location);
 
         use ast::ExpressionType::*;
+        #[allow(unused_imports)] // not unused, overrides ast::ExpressionType::None
+        use Option::None;
         match &expression.node {
             Call {
                 function,
@@ -1637,13 +1650,25 @@ impl<O: OutputStream> Compiler<O> {
                 self.mark_generator();
                 match value {
                     Some(expression) => self.compile_expression(expression)?,
-                    Option::None => self.emit(Instruction::LoadConst {
+                    None => self.emit(Instruction::LoadConst {
                         value: bytecode::Constant::None,
                     }),
                 };
                 self.emit(Instruction::YieldValue);
             }
             Await { value } => {
+                if self.ctx.func != FunctionContext::AsyncFunction {
+<<<<<<< Updated upstream
+                    return Err(CompileError {
+                        statement: None,
+                        error: CompileErrorType::InvalidAwait,
+                        location: self.current_source_location.clone(),
+                        source_path: None,
+                    });
+=======
+                    return Err(self.error(CompileErrorType::InvalidAwait));
+>>>>>>> Stashed changes
+                }
                 self.compile_expression(value)?;
                 self.emit(Instruction::GetAwaitable);
                 self.emit(Instruction::LoadConst {
@@ -1652,6 +1677,32 @@ impl<O: OutputStream> Compiler<O> {
                 self.emit(Instruction::YieldFrom);
             }
             YieldFrom { value } => {
+                match self.ctx.func {
+                    FunctionContext::NoFunction => {
+<<<<<<< Updated upstream
+                        return Err(CompileError {
+                            statement: None,
+                            error: CompileErrorType::InvalidYieldFrom,
+                            location: self.current_source_location.clone(),
+                            source_path: None,
+                        })
+                    }
+                    FunctionContext::AsyncFunction => {
+                        return Err(CompileError {
+                            statement: None,
+                            error: CompileErrorType::AsyncYieldFrom,
+                            location: self.current_source_location.clone(),
+                            source_path: None,
+                        })
+=======
+                        return Err(self.error(CompileErrorType::InvalidYieldFrom))
+                    }
+                    FunctionContext::AsyncFunction => {
+                        return Err(self.error(CompileErrorType::AsyncYieldFrom))
+>>>>>>> Stashed changes
+                    }
+                    FunctionContext::Function => {}
+                }
                 self.mark_generator();
                 self.compile_expression(value)?;
                 self.emit(Instruction::GetIter);
@@ -1670,7 +1721,7 @@ impl<O: OutputStream> Compiler<O> {
                     value: bytecode::Constant::Boolean { value: false },
                 });
             }
-            None => {
+            ast::ExpressionType::None => {
                 self.emit(Instruction::LoadConst {
                     value: bytecode::Constant::None,
                 });
diff --git a/src/error.rs b/src/error.rs
index 836e479a..2c825013 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -54,6 +54,9 @@ pub enum CompileErrorType {
     InvalidContinue,
     InvalidReturn,
     InvalidYield,
+    InvalidYieldFrom,
+    InvalidAwait,
+    AsyncYieldFrom,
 }
 
 impl CompileError {
@@ -96,6 +99,9 @@ impl fmt::Display for CompileError {
             CompileErrorType::InvalidContinue => "'continue' outside loop".to_owned(),
             CompileErrorType::InvalidReturn => "'return' outside function".to_owned(),
             CompileErrorType::InvalidYield => "'yield' outside function".to_owned(),
+            CompileErrorType::InvalidYieldFrom => "'yield from' outside function".to_owned(),
+            CompileErrorType::InvalidAwait => "'await' outside async function".to_owned(),
+            CompileErrorType::AsyncYieldFrom => "'yield from' inside async function".to_owned(),
         };
 
         if let Some(statement) = &self.statement {

From 7d6d5ff90732f9b4a49c81af8c8a4abf3752fb2a Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Thu, 5 Mar 2020 00:57:05 -0600
Subject: [PATCH 108/459] Fix syntax error tests

---
 src/compile.rs       | 29 -----------------------------
 src/error.rs         |  4 ++++
 src/output_stream.rs |  5 +++++
 src/peephole.rs      |  3 +++
 4 files changed, 12 insertions(+), 29 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index f4d088c4..29a876f5 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -562,8 +562,6 @@ impl<O: OutputStream> Compiler<O> {
                 }
                 match value {
                     Some(v) => {
-<<<<<<< Updated upstream
-=======
                         if self.ctx.func == FunctionContext::AsyncFunction
                             && self.current_output().is_generator()
                         {
@@ -572,7 +570,6 @@ impl<O: OutputStream> Compiler<O> {
                                 statement.location.clone(),
                             ));
                         }
->>>>>>> Stashed changes
                         self.compile_expression(v)?;
                     }
                     None => {
@@ -1658,16 +1655,7 @@ impl<O: OutputStream> Compiler<O> {
             }
             Await { value } => {
                 if self.ctx.func != FunctionContext::AsyncFunction {
-<<<<<<< Updated upstream
-                    return Err(CompileError {
-                        statement: None,
-                        error: CompileErrorType::InvalidAwait,
-                        location: self.current_source_location.clone(),
-                        source_path: None,
-                    });
-=======
                     return Err(self.error(CompileErrorType::InvalidAwait));
->>>>>>> Stashed changes
                 }
                 self.compile_expression(value)?;
                 self.emit(Instruction::GetAwaitable);
@@ -1679,27 +1667,10 @@ impl<O: OutputStream> Compiler<O> {
             YieldFrom { value } => {
                 match self.ctx.func {
                     FunctionContext::NoFunction => {
-<<<<<<< Updated upstream
-                        return Err(CompileError {
-                            statement: None,
-                            error: CompileErrorType::InvalidYieldFrom,
-                            location: self.current_source_location.clone(),
-                            source_path: None,
-                        })
-                    }
-                    FunctionContext::AsyncFunction => {
-                        return Err(CompileError {
-                            statement: None,
-                            error: CompileErrorType::AsyncYieldFrom,
-                            location: self.current_source_location.clone(),
-                            source_path: None,
-                        })
-=======
                         return Err(self.error(CompileErrorType::InvalidYieldFrom))
                     }
                     FunctionContext::AsyncFunction => {
                         return Err(self.error(CompileErrorType::AsyncYieldFrom))
->>>>>>> Stashed changes
                     }
                     FunctionContext::Function => {}
                 }
diff --git a/src/error.rs b/src/error.rs
index 2c825013..6f28e3a3 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -57,6 +57,7 @@ pub enum CompileErrorType {
     InvalidYieldFrom,
     InvalidAwait,
     AsyncYieldFrom,
+    AsyncReturnValue,
 }
 
 impl CompileError {
@@ -102,6 +103,9 @@ impl fmt::Display for CompileError {
             CompileErrorType::InvalidYieldFrom => "'yield from' outside function".to_owned(),
             CompileErrorType::InvalidAwait => "'await' outside async function".to_owned(),
             CompileErrorType::AsyncYieldFrom => "'yield from' inside async function".to_owned(),
+            CompileErrorType::AsyncReturnValue => {
+                "'return' with value inside async generator".to_owned()
+            }
         };
 
         if let Some(statement) = &self.statement {
diff --git a/src/output_stream.rs b/src/output_stream.rs
index fa358817..9cc49a94 100644
--- a/src/output_stream.rs
+++ b/src/output_stream.rs
@@ -7,6 +7,8 @@ pub trait OutputStream: From<CodeObject> + Into<CodeObject> {
     fn set_label(&mut self, label: Label);
     /// Mark the inner CodeObject as a generator
     fn mark_generator(&mut self);
+    /// Check to see if the inner CodeObject is a generator
+    fn is_generator(&self) -> bool;
 }
 
 pub struct CodeObjectStream {
@@ -36,4 +38,7 @@ impl OutputStream for CodeObjectStream {
     fn mark_generator(&mut self) {
         self.code.flags |= CodeFlags::IS_GENERATOR;
     }
+    fn is_generator(&self) -> bool {
+        self.code.flags.contains(CodeFlags::IS_GENERATOR)
+    }
 }
diff --git a/src/peephole.rs b/src/peephole.rs
index b8526686..67a4b9ef 100644
--- a/src/peephole.rs
+++ b/src/peephole.rs
@@ -112,6 +112,9 @@ where
     fn mark_generator(&mut self) {
         self.inner.mark_generator()
     }
+    fn is_generator(&self) -> bool {
+        self.inner.is_generator()
+    }
 }
 
 impl<O: OutputStream> OptimizationBuffer for PeepholeOptimizer<O> {

From 170719180aea8c440e471adf0dedff766d4614d3 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 13 Mar 2020 08:16:27 -0500
Subject: [PATCH 109/459] Apply review comments

---
 src/compile.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 29a876f5..8f88965b 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1551,8 +1551,6 @@ impl<O: OutputStream> Compiler<O> {
         self.set_source_location(expression.location);
 
         use ast::ExpressionType::*;
-        #[allow(unused_imports)] // not unused, overrides ast::ExpressionType::None
-        use Option::None;
         match &expression.node {
             Call {
                 function,
@@ -1647,7 +1645,7 @@ impl<O: OutputStream> Compiler<O> {
                 self.mark_generator();
                 match value {
                     Some(expression) => self.compile_expression(expression)?,
-                    None => self.emit(Instruction::LoadConst {
+                    Option::None => self.emit(Instruction::LoadConst {
                         value: bytecode::Constant::None,
                     }),
                 };

From 61dc0181e0c8d44d0099d1395aa1d58e59a30b36 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Thu, 2 Apr 2020 09:02:18 -0500
Subject: [PATCH 110/459] Move varargs information into CodeFlags

---
 src/compile.rs | 41 ++++++++++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 8f88965b..9267c900 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -14,7 +14,7 @@ use crate::symboltable::{
 };
 use itertools::Itertools;
 use num_complex::Complex64;
-use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Label, Varargs};
+use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Label};
 use rustpython_parser::{ast, parser};
 
 type BasicOutputStream = PeepholeOptimizer<CodeObjectStream>;
@@ -177,9 +177,9 @@ impl<O: OutputStream> Compiler<O> {
             Default::default(),
             0,
             Vec::new(),
-            Varargs::None,
+            None,
             Vec::new(),
-            Varargs::None,
+            None,
             self.source_path.clone().unwrap(),
             line_number,
             obj_name,
@@ -686,14 +686,29 @@ impl<O: OutputStream> Compiler<O> {
             flags |= bytecode::CodeFlags::HAS_KW_ONLY_DEFAULTS;
         }
 
+        let mut compile_varargs = |va: &ast::Varargs, flag| match va {
+            ast::Varargs::None => None,
+            ast::Varargs::Unnamed => {
+                flags |= flag;
+                None
+            }
+            ast::Varargs::Named(name) => {
+                flags |= flag;
+                Some(name.arg.clone())
+            }
+        };
+
+        let varargs_name = compile_varargs(&args.vararg, bytecode::CodeFlags::HAS_VARARGS);
+        let varkeywords_name = compile_varargs(&args.kwarg, bytecode::CodeFlags::HAS_VARKEYWORDS);
+
         let line_number = self.get_source_line_number();
         self.push_output(CodeObject::new(
             flags,
             args.posonlyargs_count,
             args.args.iter().map(|a| a.arg.clone()).collect(),
-            compile_varargs(&args.vararg),
+            varargs_name,
             args.kwonlyargs.iter().map(|a| a.arg.clone()).collect(),
-            compile_varargs(&args.kwarg),
+            varkeywords_name,
             self.source_path.clone().unwrap(),
             line_number,
             name.to_owned(),
@@ -970,9 +985,9 @@ impl<O: OutputStream> Compiler<O> {
             Default::default(),
             0,
             vec![],
-            Varargs::None,
+            None,
             vec![],
-            Varargs::None,
+            None,
             self.source_path.clone().unwrap(),
             line_number,
             name.to_owned(),
@@ -1920,9 +1935,9 @@ impl<O: OutputStream> Compiler<O> {
             Default::default(),
             1,
             vec![".0".to_owned()],
-            Varargs::None,
+            None,
             vec![],
-            Varargs::None,
+            None,
             self.source_path.clone().unwrap(),
             line_number,
             name.clone(),
@@ -2223,14 +2238,6 @@ fn compile_location(location: &ast::Location) -> bytecode::Location {
     bytecode::Location::new(location.row(), location.column())
 }
 
-fn compile_varargs(varargs: &ast::Varargs) -> bytecode::Varargs {
-    match varargs {
-        ast::Varargs::None => bytecode::Varargs::None,
-        ast::Varargs::Unnamed => bytecode::Varargs::Unnamed,
-        ast::Varargs::Named(param) => bytecode::Varargs::Named(param.arg.clone()),
-    }
-}
-
 fn compile_conversion_flag(conversion_flag: ast::ConversionFlag) -> bytecode::ConversionFlag {
     match conversion_flag {
         ast::ConversionFlag::Ascii => bytecode::ConversionFlag::Ascii,

From 330fdce246865b8cceaf0c23c56686670ec94e4b Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 5 Feb 2020 01:21:42 +0000
Subject: [PATCH 111/459] Add "incognito" compilation

---
 src/compile.rs | 56 ++++++++++++++++++++++++++++++++++----------------
 1 file changed, 38 insertions(+), 18 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 8f88965b..448524df 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -30,7 +30,26 @@ struct Compiler<O: OutputStream = BasicOutputStream> {
     current_source_location: ast::Location,
     current_qualified_path: Option<String>,
     ctx: CompileContext,
-    optimize: u8,
+    opts: CompileOpts,
+}
+
+#[derive(Debug, Clone)]
+pub struct CompileOpts {
+    /// How optimized the bytecode output should be; any optimize > 0 does
+    /// not emit assert statements
+    pub optimize: u8,
+    /// Whether introspection on defined functions/other items should be allowed;
+    /// e.g. when true, `func.__globals__` throws an error (where func is a function defined
+    /// in an incognito context)
+    pub incognito: bool,
+}
+impl Default for CompileOpts {
+    fn default() -> Self {
+        CompileOpts {
+            optimize: 0,
+            incognito: false,
+        }
+    }
 }
 
 #[derive(Clone, Copy)]
@@ -60,20 +79,20 @@ pub fn compile(
     source: &str,
     mode: Mode,
     source_path: String,
-    optimize: u8,
+    opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
     match mode {
         Mode::Exec => {
             let ast = parser::parse_program(source)?;
-            compile_program(ast, source_path, optimize)
+            compile_program(ast, source_path.clone(), opts)
         }
         Mode::Eval => {
             let statement = parser::parse_statement(source)?;
-            compile_statement_eval(statement, source_path, optimize)
+            compile_statement_eval(statement, source_path.clone(), opts)
         }
         Mode::Single => {
             let ast = parser::parse_program(source)?;
-            compile_program_single(ast, source_path, optimize)
+            compile_program_single(ast, source_path.clone(), opts)
         }
     }
 }
@@ -81,10 +100,10 @@ pub fn compile(
 /// A helper function for the shared code of the different compile functions
 fn with_compiler(
     source_path: String,
-    optimize: u8,
+    opts: CompileOpts,
     f: impl FnOnce(&mut Compiler) -> CompileResult<()>,
 ) -> CompileResult<CodeObject> {
-    let mut compiler = Compiler::new(optimize);
+    let mut compiler = Compiler::new(opts);
     compiler.source_path = Some(source_path);
     compiler.push_new_code_object("<module>".to_owned());
     f(&mut compiler)?;
@@ -97,9 +116,9 @@ fn with_compiler(
 pub fn compile_program(
     ast: ast::Program,
     source_path: String,
-    optimize: u8,
+    opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    with_compiler(source_path, optimize, |compiler| {
+    with_compiler(source_path, opts, |compiler| {
         let symbol_table = make_symbol_table(&ast)?;
         compiler.compile_program(&ast, symbol_table)
     })
@@ -109,9 +128,9 @@ pub fn compile_program(
 pub fn compile_statement_eval(
     statement: Vec<ast::Statement>,
     source_path: String,
-    optimize: u8,
+    opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    with_compiler(source_path, optimize, |compiler| {
+    with_compiler(source_path, opts, |compiler| {
         let symbol_table = statements_to_symbol_table(&statement)?;
         compiler.compile_statement_eval(&statement, symbol_table)
     })
@@ -121,9 +140,9 @@ pub fn compile_statement_eval(
 pub fn compile_program_single(
     ast: ast::Program,
     source_path: String,
-    optimize: u8,
+    opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    with_compiler(source_path, optimize, |compiler| {
+    with_compiler(source_path, opts, |compiler| {
         let symbol_table = make_symbol_table(&ast)?;
         compiler.compile_program_single(&ast, symbol_table)
     })
@@ -134,12 +153,12 @@ where
     O: OutputStream,
 {
     fn default() -> Self {
-        Compiler::new(0)
+        Compiler::new(CompileOpts::default())
     }
 }
 
 impl<O: OutputStream> Compiler<O> {
-    fn new(optimize: u8) -> Self {
+    fn new(opts: CompileOpts) -> Self {
         Compiler {
             output_stack: Vec::new(),
             symbol_table_stack: Vec::new(),
@@ -151,7 +170,7 @@ impl<O: OutputStream> Compiler<O> {
                 in_loop: false,
                 func: FunctionContext::NoFunction,
             },
-            optimize,
+            opts,
         }
     }
 
@@ -167,7 +186,8 @@ impl<O: OutputStream> Compiler<O> {
         }
     }
 
-    fn push_output(&mut self, code: CodeObject) {
+    fn push_output(&mut self, mut code: CodeObject) {
+        code.incognito = self.opts.incognito;
         self.output_stack.push(code.into());
     }
 
@@ -518,7 +538,7 @@ impl<O: OutputStream> Compiler<O> {
             } => self.compile_class_def(name, body, bases, keywords, decorator_list)?,
             Assert { test, msg } => {
                 // if some flag, ignore all assert statements!
-                if self.optimize == 0 {
+                if self.opts.optimize == 0 {
                     let end_label = self.new_label();
                     self.compile_jump_if(test, true, end_label)?;
                     self.emit(Instruction::LoadName {

From a499fb775ac0c75c28ec2b24ab3fe96a6621753d Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 7 Feb 2020 21:31:58 +0000
Subject: [PATCH 112/459] Make `from __future__` imports a syntactic construct
 in the compiler

---
 src/compile.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++
 src/error.rs   |  8 ++++++++
 2 files changed, 52 insertions(+)

diff --git a/src/compile.rs b/src/compile.rs
index 448524df..bceee87f 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -29,6 +29,7 @@ struct Compiler<O: OutputStream = BasicOutputStream> {
     source_path: Option<String>,
     current_source_location: ast::Location,
     current_qualified_path: Option<String>,
+    done_with_future_stmts: bool,
     ctx: CompileContext,
     opts: CompileOpts,
 }
@@ -166,6 +167,7 @@ impl<O: OutputStream> Compiler<O> {
             source_path: None,
             current_source_location: ast::Location::default(),
             current_qualified_path: None,
+            done_with_future_stmts: false,
             ctx: CompileContext {
                 in_loop: false,
                 func: FunctionContext::NoFunction,
@@ -334,6 +336,16 @@ impl<O: OutputStream> Compiler<O> {
         self.set_source_location(statement.location);
         use ast::StatementType::*;
 
+        match &statement.node {
+            // we do this here because `from __future__` still executes that `from` statement at runtime,
+            // we still need to compile the ImportFrom down below
+            ImportFrom { module, names, .. } if module.as_deref() == Some("__future__") => {
+                self.compile_future_features(&names)?
+            }
+            // if we find any other statement, stop accepting future statements
+            _ => self.done_with_future_stmts = true,
+        }
+
         match &statement.node {
             Import { names } => {
                 // import a, b, c as d
@@ -2132,6 +2144,38 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
+    fn compile_future_features(
+        &mut self,
+        features: &[ast::ImportSymbol],
+    ) -> Result<(), CompileError> {
+        if self.done_with_future_stmts {
+            return Err(CompileError {
+                error: CompileErrorType::InvalidFuturePlacement,
+                location: self.current_source_location.clone(),
+                source_path: self.source_path.clone(),
+                statement: None,
+            });
+        }
+        for feature in features {
+            match &*feature.symbol {
+                // Python 3 features; we've already implemented them by default
+                "nested_scopes" | "generators" | "division" | "absolute_import"
+                | "with_statement" | "print_function" | "unicode_literals" => {}
+                // "generator_stop" => {}
+                // "annotations" => {}
+                other => {
+                    return Err(CompileError {
+                        error: CompileErrorType::InvalidFutureFeature(other.to_owned()),
+                        location: self.current_source_location.clone(),
+                        source_path: self.source_path.clone(),
+                        statement: None,
+                    })
+                }
+            }
+        }
+        Ok(())
+    }
+
     // Scope helpers:
     fn enter_scope(&mut self) {
         // println!("Enter scope {:?}", self.symbol_table_stack);
diff --git a/src/error.rs b/src/error.rs
index 6f28e3a3..85bc9f46 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -58,6 +58,8 @@ pub enum CompileErrorType {
     InvalidAwait,
     AsyncYieldFrom,
     AsyncReturnValue,
+    InvalidFuturePlacement,
+    InvalidFutureFeature(String),
 }
 
 impl CompileError {
@@ -106,6 +108,12 @@ impl fmt::Display for CompileError {
             CompileErrorType::AsyncReturnValue => {
                 "'return' with value inside async generator".to_owned()
             }
+            CompileErrorType::InvalidFuturePlacement => {
+                "from __future__ imports must occur at the beginning of the file".to_owned()
+            }
+            CompileErrorType::InvalidFutureFeature(feat) => {
+                format!("future feature {} is not defined", feat)
+            }
         };
 
         if let Some(statement) = &self.statement {

From ed09df8223d740737889ed6adbee800768d82a8e Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 8 Feb 2020 23:23:03 -0600
Subject: [PATCH 113/459] Make incognito a CodeFlag bit

---
 src/compile.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index bceee87f..e602159f 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -189,7 +189,8 @@ impl<O: OutputStream> Compiler<O> {
     }
 
     fn push_output(&mut self, mut code: CodeObject) {
-        code.incognito = self.opts.incognito;
+        code.flags
+            .set(bytecode::CodeFlags::INCOGNITO, self.opts.incognito);
         self.output_stack.push(code.into());
     }
 

From 9ac8969e0d73d001fee300942a4b63bb98617b1e Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 28 Feb 2020 21:00:40 -0600
Subject: [PATCH 114/459] Remove incognito from vm

---
 src/compile.rs | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index e602159f..8e9aad60 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -39,17 +39,10 @@ pub struct CompileOpts {
     /// How optimized the bytecode output should be; any optimize > 0 does
     /// not emit assert statements
     pub optimize: u8,
-    /// Whether introspection on defined functions/other items should be allowed;
-    /// e.g. when true, `func.__globals__` throws an error (where func is a function defined
-    /// in an incognito context)
-    pub incognito: bool,
 }
 impl Default for CompileOpts {
     fn default() -> Self {
-        CompileOpts {
-            optimize: 0,
-            incognito: false,
-        }
+        CompileOpts { optimize: 0 }
     }
 }
 
@@ -188,9 +181,7 @@ impl<O: OutputStream> Compiler<O> {
         }
     }
 
-    fn push_output(&mut self, mut code: CodeObject) {
-        code.flags
-            .set(bytecode::CodeFlags::INCOGNITO, self.opts.incognito);
+    fn push_output(&mut self, code: CodeObject) {
         self.output_stack.push(code.into());
     }
 

From 8df7d73211610cb626e8222ce6d88ba069848ab2 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 3 Apr 2020 12:33:49 -0500
Subject: [PATCH 115/459] Fix clippy lints

---
 src/compile.rs | 20 +++++---------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 8e9aad60..413c9c43 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -78,15 +78,15 @@ pub fn compile(
     match mode {
         Mode::Exec => {
             let ast = parser::parse_program(source)?;
-            compile_program(ast, source_path.clone(), opts)
+            compile_program(ast, source_path, opts)
         }
         Mode::Eval => {
             let statement = parser::parse_statement(source)?;
-            compile_statement_eval(statement, source_path.clone(), opts)
+            compile_statement_eval(statement, source_path, opts)
         }
         Mode::Single => {
             let ast = parser::parse_program(source)?;
-            compile_program_single(ast, source_path.clone(), opts)
+            compile_program_single(ast, source_path, opts)
         }
     }
 }
@@ -2141,12 +2141,7 @@ impl<O: OutputStream> Compiler<O> {
         features: &[ast::ImportSymbol],
     ) -> Result<(), CompileError> {
         if self.done_with_future_stmts {
-            return Err(CompileError {
-                error: CompileErrorType::InvalidFuturePlacement,
-                location: self.current_source_location.clone(),
-                source_path: self.source_path.clone(),
-                statement: None,
-            });
+            return Err(self.error(CompileErrorType::InvalidFuturePlacement));
         }
         for feature in features {
             match &*feature.symbol {
@@ -2156,12 +2151,7 @@ impl<O: OutputStream> Compiler<O> {
                 // "generator_stop" => {}
                 // "annotations" => {}
                 other => {
-                    return Err(CompileError {
-                        error: CompileErrorType::InvalidFutureFeature(other.to_owned()),
-                        location: self.current_source_location.clone(),
-                        source_path: self.source_path.clone(),
-                        statement: None,
-                    })
+                    return Err(self.error(CompileErrorType::InvalidFutureFeature(other.to_owned())))
                 }
             }
         }

From c27abf862f44aed1dba8f558cd31a2bc941dc294 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 16 Mar 2020 20:56:45 -0500
Subject: [PATCH 116/459] Fix some clippy lints that were 'allow'ed

---
 src/symboltable.rs | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index d08aa4f2..0ad36333 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -741,7 +741,6 @@ impl SymbolTableBuilder {
         Ok(())
     }
 
-    #[allow(clippy::single_match)]
     fn register_name(&mut self, name: &str, role: SymbolUsage) -> SymbolTableResult {
         let scope_depth = self.tables.len();
         let table = self.tables.last_mut().unwrap();
@@ -777,13 +776,11 @@ impl SymbolTableBuilder {
 
         // Some more checks:
         match role {
-            SymbolUsage::Nonlocal => {
-                if scope_depth < 2 {
-                    return Err(SymbolTableError {
-                        error: format!("cannot define nonlocal '{}' at top level.", name),
-                        location,
-                    });
-                }
+            SymbolUsage::Nonlocal if scope_depth < 2 => {
+                return Err(SymbolTableError {
+                    error: format!("cannot define nonlocal '{}' at top level.", name),
+                    location,
+                })
             }
             _ => {
                 // Ok!

From aba7359b3620c1519163f909469841b1bf712db1 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 24 Apr 2020 15:14:58 -0500
Subject: [PATCH 117/459] Disable the unpack optimization

---
 src/peephole.rs               |  2 +-
 src/peephole/optimizations.rs | 47 ++++++++++++++++++-----------------
 2 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/src/peephole.rs b/src/peephole.rs
index 67a4b9ef..aaec0470 100644
--- a/src/peephole.rs
+++ b/src/peephole.rs
@@ -92,7 +92,7 @@ impl<O: OutputStream> PeepholeOptimizer<O> {
     }
 
     fn optimize(&mut self) {
-        apply_optimizations!(self, operator, unpack);
+        apply_optimizations!(self, operator /* , unpack */);
     }
 }
 
diff --git a/src/peephole/optimizations.rs b/src/peephole/optimizations.rs
index 28bac862..461baa7c 100644
--- a/src/peephole/optimizations.rs
+++ b/src/peephole/optimizations.rs
@@ -73,26 +73,27 @@ pub fn operator(buf: &mut impl OptimizationBuffer) {
     }
 }
 
-pub fn unpack(buf: &mut impl OptimizationBuffer) {
-    let (instruction, meta) = buf.pop();
-    if let Instruction::UnpackSequence { size } = instruction {
-        let (arg, arg_meta) = buf.pop();
-        match arg {
-            Instruction::BuildTuple {
-                size: tup_size,
-                unpack,
-            } if !unpack && tup_size == size => {
-                buf.emit(
-                    Instruction::Reverse { amount: size },
-                    vec![arg_meta, meta].into(),
-                );
-            }
-            arg => {
-                buf.emit(arg, arg_meta);
-                buf.emit(instruction, meta);
-            }
-        }
-    } else {
-        buf.emit(instruction, meta)
-    }
-}
+// TODO: make a version of this that doesn't miscompile `a, b = (1, 2) if True else (3, 4)`
+// pub fn unpack(buf: &mut impl OptimizationBuffer) {
+//     let (instruction, meta) = buf.pop();
+//     if let Instruction::UnpackSequence { size } = instruction {
+//         let (arg, arg_meta) = buf.pop();
+//         match arg {
+//             Instruction::BuildTuple {
+//                 size: tup_size,
+//                 unpack,
+//             } if !unpack && tup_size == size => {
+//                 buf.emit(
+//                     Instruction::Reverse { amount: size },
+//                     vec![arg_meta, meta].into(),
+//                 );
+//             }
+//             arg => {
+//                 buf.emit(arg, arg_meta);
+//                 buf.emit(instruction, meta);
+//             }
+//         }
+//     } else {
+//         buf.emit(instruction, meta)
+//     }
+// }

From e85baf4dfb32ce9d17ae48af6f0548fabb6b9a9f Mon Sep 17 00:00:00 2001
From: HyeockJinKim <kherootz@gmail.com>
Date: Sat, 25 Apr 2020 13:59:54 +0900
Subject: [PATCH 118/459] Remove unused variable

Removed unused `is_param` variable for symbol.

Closes RustPython#1879
---
 src/symboltable.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index d08aa4f2..02bbf1f6 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -95,7 +95,6 @@ pub struct Symbol {
     pub name: String,
     // pub table: SymbolTableRef,
     pub scope: SymbolScope,
-    pub is_param: bool,
     pub is_referenced: bool,
     pub is_assigned: bool,
     pub is_parameter: bool,
@@ -108,7 +107,6 @@ impl Symbol {
             name: name.to_owned(),
             // table,
             scope: SymbolScope::Unknown,
-            is_param: false,
             is_referenced: false,
             is_assigned: false,
             is_parameter: false,

From a77e0b858797697975e203181bfd9587c7672e12 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 13 May 2020 01:42:18 +0900
Subject: [PATCH 119/459] Distinguish MultipleStarArgs and InvalidStarArgs
 compile errors

---
 src/compile.rs | 8 ++------
 src/error.rs   | 9 +++++++--
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 42c78b28..df543eb7 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1374,7 +1374,7 @@ impl<O: OutputStream> Compiler<O> {
                 for (i, element) in elements.iter().enumerate() {
                     if let ast::ExpressionType::Starred { .. } = &element.node {
                         if seen_star {
-                            return Err(self.error(CompileErrorType::StarArgs));
+                            return Err(self.error(CompileErrorType::MultipleStarArgs));
                         } else {
                             seen_star = true;
                             self.emit(Instruction::UnpackEx {
@@ -1782,11 +1782,7 @@ impl<O: OutputStream> Compiler<O> {
                 self.compile_comprehension(kind, generators)?;
             }
             Starred { .. } => {
-                return Err(
-                    self.error(CompileErrorType::SyntaxError(std::string::String::from(
-                        "Invalid starred expression",
-                    ))),
-                );
+                return Err(self.error(CompileErrorType::InvalidStarExpr));
             }
             IfExpression { test, body, orelse } => {
                 let no_label = self.new_label();
diff --git a/src/error.rs b/src/error.rs
index 85bc9f46..38e54300 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -47,7 +47,9 @@ pub enum CompileErrorType {
     Parse(ParseErrorType),
     SyntaxError(String),
     /// Multiple `*` detected
-    StarArgs,
+    MultipleStarArgs,
+    /// Misplaced `*` expression
+    InvalidStarExpr,
     /// Break statement outside of loop.
     InvalidBreak,
     /// Continue statement outside of loop.
@@ -97,7 +99,10 @@ impl fmt::Display for CompileError {
             CompileErrorType::ExpectExpr => "Expecting expression, got statement".to_owned(),
             CompileErrorType::Parse(err) => err.to_string(),
             CompileErrorType::SyntaxError(err) => err.to_string(),
-            CompileErrorType::StarArgs => "Two starred expressions in assignment".to_owned(),
+            CompileErrorType::MultipleStarArgs => {
+                "two starred expressions in assignment".to_owned()
+            }
+            CompileErrorType::InvalidStarExpr => "can't use starred expression here".to_owned(),
             CompileErrorType::InvalidBreak => "'break' outside loop".to_owned(),
             CompileErrorType::InvalidContinue => "'continue' outside loop".to_owned(),
             CompileErrorType::InvalidReturn => "'return' outside function".to_owned(),

From 0d139f9942ae4f75a3017a4b8b078575bd582031 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 13 May 2020 01:43:09 +0900
Subject: [PATCH 120/459] comprehension starred expression compatibility

---
 src/compile.rs | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index df543eb7..4a8ee902 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -2027,21 +2027,33 @@ impl<O: OutputStream> Compiler<O> {
             }
         }
 
+        let mut compile_element = |element| {
+            self.compile_expression(element).map_err(|e| {
+                if matches!(e.error, CompileErrorType::InvalidStarExpr) {
+                    self.error(CompileErrorType::SyntaxError(
+                        "iterable unpacking cannot be used in comprehension".to_owned(),
+                    ))
+                } else {
+                    e
+                }
+            })
+        };
+
         match kind {
             ast::ComprehensionKind::GeneratorExpression { element } => {
-                self.compile_expression(element)?;
+                compile_element(element)?;
                 self.mark_generator();
                 self.emit(Instruction::YieldValue);
                 self.emit(Instruction::Pop);
             }
             ast::ComprehensionKind::List { element } => {
-                self.compile_expression(element)?;
+                compile_element(element)?;
                 self.emit(Instruction::ListAppend {
                     i: 1 + generators.len(),
                 });
             }
             ast::ComprehensionKind::Set { element } => {
-                self.compile_expression(element)?;
+                compile_element(element)?;
                 self.emit(Instruction::SetAdd {
                     i: 1 + generators.len(),
                 });

From 3f1b9c6096432435349aa3915e1f12709dbdae3a Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 13 May 2020 01:46:37 +0900
Subject: [PATCH 121/459] Fix SyntaxError visualize order: message first
 location later

---
 src/error.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/error.rs b/src/error.rs
index 38e54300..86635bc9 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -125,7 +125,7 @@ impl fmt::Display for CompileError {
             if self.location.column() > 0 {
                 if let Some(line) = statement.lines().nth(self.location.row() - 1) {
                     // visualize the error, when location and statement are provided
-                    return write!(f, "\n{}\n{}", line, self.location.visualize(&error_desc));
+                    return write!(f, "{}", self.location.visualize(line, &error_desc));
                 }
             }
         }

From 1257331101ff4b9c765983b7bb42525c560da4be Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 13 May 2020 01:48:40 +0900
Subject: [PATCH 122/459] Fix unpack_ex related messages

---
 src/compile.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index 4a8ee902..53aa3626 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1399,7 +1399,14 @@ impl<O: OutputStream> Compiler<O> {
                     }
                 }
             }
-            _ => return Err(self.error(CompileErrorType::Assign(target.name()))),
+            _ => {
+                return Err(self.error(match target.node {
+                    ast::ExpressionType::Starred { .. } => CompileErrorType::SyntaxError(
+                        "starred assignment target must be in a list or tuple".to_owned(),
+                    ),
+                    _ => CompileErrorType::Assign(target.name()),
+                }))
+            }
         }
 
         Ok(())

From 097b6a1889562406a746d5a64bd8839fcf41542a Mon Sep 17 00:00:00 2001
From: TheAnyKey <32773684+TheAnyKey@users.noreply.github.com>
Date: Mon, 1 Jun 2020 11:22:15 +0200
Subject: [PATCH 123/459] Implement Py38 named expression (PEP 572) (#1934)

* Initial implementation of named expression and  import  according CPython tests

* added new instruction with inversed evaluation order for dict comprehension, in other cases use regular evaluation order

* added further aspects to implementation, cleaned up, imported test from CPython

* implemented first parts of scoping enhancement and extended checks

* completion of name resolution ongoing, now more test passing, still warinings and cleanup required

* further optimization of name resolution in nested scopes

* Initialize the vm with imports from _io instead of io

* Add the OpenBSD support that I am smart enough to

* adapted grammer to full support, most test are passing now, esp. all invalids are passed. Cleaned up in symboltable

* more conditional compiling, this time for errors

* rustfmt was not pleased

* premature push, whoops

* Add expected_failure result type to jsontests

* Initial implementation of named expression and  import  according CPython tests

* added new instruction with inversed evaluation order for dict comprehension, in other cases use regular evaluation order

* added further aspects to implementation, cleaned up, imported test from CPython

* implemented first parts of scoping enhancement and extended checks

* completion of name resolution ongoing, now more test passing, still warinings and cleanup required

* further optimization of name resolution in nested scopes

* adapted grammer to full support, most test are passing now, esp. all invalids are passed. Cleaned up in symboltable

* Fixed nameing convention violation and removed unnecessary information from symbol resolution. Added some more comments.

* Fixed nameing convention violation and removed unnecessary information from symbol resolution. Added some more comments.

Co-authored-by: Noah <33094578+coolreader18@users.noreply.github.com>
Co-authored-by: Reuben Staley <lighthousemaniac@gmail.com>
---
 src/compile.rs     |  11 ++-
 src/symboltable.rs | 196 ++++++++++++++++++++++++++++++++++-----------
 2 files changed, 157 insertions(+), 50 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 53aa3626..b2273bfa 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1804,6 +1804,12 @@ impl<O: OutputStream> Compiler<O> {
                 // End
                 self.set_label(end_label);
             }
+
+            NamedExpression { left, right } => {
+                self.compile_expression(right)?;
+                self.emit(Instruction::Duplicate);
+                self.compile_store(left)?;
+            }
         }
         Ok(())
     }
@@ -2066,10 +2072,11 @@ impl<O: OutputStream> Compiler<O> {
                 });
             }
             ast::ComprehensionKind::Dict { key, value } => {
-                self.compile_expression(value)?;
+                // changed evaluation order for Py38 named expression PEP 572
                 self.compile_expression(key)?;
+                self.compile_expression(value)?;
 
-                self.emit(Instruction::MapAdd {
+                self.emit(Instruction::MapAddRev {
                     i: 1 + generators.len(),
                 });
             }
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 22acbe1a..ca9594e7 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -66,6 +66,7 @@ pub enum SymbolTableType {
     Module,
     Class,
     Function,
+    Comprehension,
 }
 
 impl fmt::Display for SymbolTableType {
@@ -74,6 +75,7 @@ impl fmt::Display for SymbolTableType {
             SymbolTableType::Module => write!(f, "module"),
             SymbolTableType::Class => write!(f, "class"),
             SymbolTableType::Function => write!(f, "function"),
+            SymbolTableType::Comprehension => write!(f, "comprehension"),
         }
     }
 }
@@ -99,6 +101,10 @@ pub struct Symbol {
     pub is_assigned: bool,
     pub is_parameter: bool,
     pub is_free: bool,
+
+    // indicates if the symbol gets a value assigned by a named expression in a comprehension
+    // this is required to correct the scope in the analysis.
+    pub is_assign_namedexpr_in_comprehension: bool,
 }
 
 impl Symbol {
@@ -111,6 +117,7 @@ impl Symbol {
             is_assigned: false,
             is_parameter: false,
             is_free: false,
+            is_assign_namedexpr_in_comprehension: false,
         }
     }
 
@@ -193,72 +200,138 @@ impl<'a> SymbolTableAnalyzer<'a> {
         for sub_table in sub_tables {
             self.analyze_symbol_table(sub_table)?;
         }
-        let (symbols, _) = self.tables.pop().unwrap();
+        let (symbols, st_typ) = self.tables.pop().unwrap();
 
         // Analyze symbols:
         for symbol in symbols.values_mut() {
-            self.analyze_symbol(symbol)?;
+            self.analyze_symbol(symbol, st_typ)?;
         }
-
         Ok(())
     }
 
-    fn analyze_symbol(&self, symbol: &mut Symbol) -> SymbolTableResult {
-        match symbol.scope {
-            SymbolScope::Nonlocal => {
-                // check if name is defined in parent table!
-                let parent_symbol_table = self.tables.last();
-                // symbol.table.borrow().parent.clone();
-
-                if let Some((symbols, _)) = parent_symbol_table {
-                    let scope_depth = self.tables.len();
-                    if !symbols.contains_key(&symbol.name) || scope_depth < 2 {
+    fn analyze_symbol(
+        &mut self,
+        symbol: &mut Symbol,
+        curr_st_typ: SymbolTableType,
+    ) -> SymbolTableResult {
+        if symbol.is_assign_namedexpr_in_comprehension
+            && curr_st_typ == SymbolTableType::Comprehension
+        {
+            self.analyze_symbol_comprehension(symbol, 0)?
+        } else {
+            match symbol.scope {
+                SymbolScope::Nonlocal => {
+                    // check if name is defined in parent table!
+                    let parent_symbol_table = self.tables.last();
+                    if let Some((symbols, _)) = parent_symbol_table {
+                        let scope_depth = self.tables.len();
+                        if !symbols.contains_key(&symbol.name) || scope_depth < 2 {
+                            return Err(SymbolTableError {
+                                error: format!("no binding for nonlocal '{}' found", symbol.name),
+                                location: Default::default(),
+                            });
+                        }
+                    } else {
                         return Err(SymbolTableError {
-                            error: format!("no binding for nonlocal '{}' found", symbol.name),
+                            error: format!(
+                                "nonlocal {} defined at place without an enclosing scope",
+                                symbol.name
+                            ),
                             location: Default::default(),
                         });
                     }
-                } else {
-                    return Err(SymbolTableError {
-                        error: format!(
-                            "nonlocal {} defined at place without an enclosing scope",
-                            symbol.name
-                        ),
-                        location: Default::default(),
-                    });
+                }
+                SymbolScope::Global => {
+                    // TODO: add more checks for globals?
+                }
+                SymbolScope::Local => {
+                    // all is well
+                }
+                SymbolScope::Unknown => {
+                    // Try hard to figure out what the scope of this symbol is.
+                    self.analyze_unknown_symbol(symbol);
                 }
             }
-            SymbolScope::Global => {
-                // TODO: add more checks for globals?
-            }
-            SymbolScope::Local => {
-                // all is well
+        }
+        Ok(())
+    }
+
+    fn analyze_unknown_symbol(&self, symbol: &mut Symbol) {
+        if symbol.is_assigned || symbol.is_parameter {
+            symbol.scope = SymbolScope::Local;
+        } else {
+            // Interesting stuff about the __class__ variable:
+            // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
+            let found_in_outer_scope = symbol.name == "__class__"
+                || self.tables.iter().skip(1).any(|(symbols, typ)| {
+                    *typ != SymbolTableType::Class && symbols.contains_key(&symbol.name)
+                });
+
+            if found_in_outer_scope {
+                // Symbol is in some outer scope.
+                symbol.is_free = true;
+            } else if self.tables.is_empty() {
+                // Don't make assumptions when we don't know.
+                symbol.scope = SymbolScope::Unknown;
+            } else {
+                // If there are scopes above we can assume global.
+                symbol.scope = SymbolScope::Global;
             }
-            SymbolScope::Unknown => {
-                // Try hard to figure out what the scope of this symbol is.
+        }
+    }
 
-                if symbol.is_assigned || symbol.is_parameter {
-                    symbol.scope = SymbolScope::Local;
-                } else {
-                    // Interesting stuff about the __class__ variable:
-                    // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
-                    let found_in_outer_scope = symbol.name == "__class__"
-                        || self.tables.iter().skip(1).any(|(symbols, typ)| {
-                            *typ != SymbolTableType::Class && symbols.contains_key(&symbol.name)
-                        });
+    // Implements the symbol analysis and scope extension for names
+    // assigned by a named expression in a comprehension. See:
+    // https://github.com/python/cpython/blob/7b78e7f9fd77bb3280ee39fb74b86772a7d46a70/Python/symtable.c#L1435
+    fn analyze_symbol_comprehension(
+        &mut self,
+        symbol: &mut Symbol,
+        parent_offset: usize,
+    ) -> SymbolTableResult {
+        // TODO: quite C-ish way to implement the iteration
+        // when this is called, we expect to be in the direct parent scope of the scope that contains 'symbol'
+        let offs = self.tables.len() - 1 - parent_offset;
+        let last = self.tables.get_mut(offs).unwrap();
+        let symbols = &mut last.0;
+        let table_type = last.1;
+
+        match table_type {
+            SymbolTableType::Module => {
+                symbol.scope = SymbolScope::Global;
+            }
+            SymbolTableType::Class => {}
+            SymbolTableType::Function => {
+                if let Some(parent_symbol) = symbols.get_mut(&symbol.name) {
+                    if let SymbolScope::Unknown = parent_symbol.scope {
+                        parent_symbol.is_assigned = true; // this information is new, as the asignment is done in inner scope
+                        self.analyze_unknown_symbol(symbol);
+                    }
 
-                    if found_in_outer_scope {
-                        // Symbol is in some outer scope.
-                        symbol.is_free = true;
-                    } else if self.tables.is_empty() {
-                        // Don't make assumptions when we don't know.
-                        symbol.scope = SymbolScope::Unknown;
-                    } else {
-                        // If there are scopes above we can assume global.
-                        symbol.scope = SymbolScope::Global;
+                    match symbol.scope {
+                        SymbolScope::Global => {
+                            symbol.scope = SymbolScope::Global;
+                        }
+                        _ => {
+                            symbol.scope = SymbolScope::Nonlocal;
+                        }
                     }
                 }
             }
+            SymbolTableType::Comprehension => {
+                // TODO check for conflicts - requires more context information about variables
+                match symbols.get_mut(&symbol.name) {
+                    Some(parent_symbol) => {
+                        parent_symbol.is_assigned = true; // more checks are required
+                    }
+                    None => {
+                        let cloned_sym = symbol.clone();
+
+                        last.0.insert(cloned_sym.name.to_owned(), cloned_sym);
+                    }
+                }
+
+                self.analyze_symbol_comprehension(symbol, parent_offset + 1)?;
+            }
         }
         Ok(())
     }
@@ -271,6 +344,7 @@ enum SymbolUsage {
     Used,
     Assigned,
     Parameter,
+    AssignedNamedExprInCompr,
 }
 
 #[derive(Default)]
@@ -602,7 +676,7 @@ impl SymbolTableBuilder {
 
                 self.enter_scope(
                     scope_name,
-                    SymbolTableType::Function,
+                    SymbolTableType::Comprehension,
                     expression.location.row(),
                 );
 
@@ -679,6 +753,28 @@ impl SymbolTableBuilder {
                 self.scan_expression(body, &ExpressionContext::Load)?;
                 self.scan_expression(orelse, &ExpressionContext::Load)?;
             }
+
+            NamedExpression { left, right } => {
+                self.scan_expression(right, &ExpressionContext::Load)?;
+
+                // special handling for assigned identifier in named expressions
+                // that are used in comprehensions. This required to correctly
+                // propagate the scope of the named assigned named and not to
+                // propagate inner names.
+                if let Identifier { name } = &left.node {
+                    let table = self.tables.last().unwrap();
+                    if table.typ == SymbolTableType::Comprehension {
+                        self.register_name(name, SymbolUsage::AssignedNamedExprInCompr)?;
+                    } else {
+                        // omit one recursion. When the handling of an store changes for
+                        // Identifiers this needs adapted - more forward safe would be
+                        // calling scan_expression directly.
+                        self.register_name(name, SymbolUsage::Assigned)?;
+                    }
+                } else {
+                    self.scan_expression(left, &ExpressionContext::Store)?;
+                }
+            }
         }
         Ok(())
     }
@@ -810,6 +906,10 @@ impl SymbolTableBuilder {
             SymbolUsage::Assigned => {
                 symbol.is_assigned = true;
             }
+            SymbolUsage::AssignedNamedExprInCompr => {
+                symbol.is_assigned = true;
+                symbol.is_assign_namedexpr_in_comprehension = true;
+            }
             SymbolUsage::Global => {
                 if let SymbolScope::Unknown = symbol.scope {
                     symbol.scope = SymbolScope::Global;

From 50c0b991366697e40bc64d50d212c78d4aee33c6 Mon Sep 17 00:00:00 2001
From: TheAnyKey <ich@simonroth.de>
Date: Mon, 1 Jun 2020 22:47:28 +0000
Subject: [PATCH 124/459] fixed scoping issue for nonlocals - now also indirect
 outer scopes are supported

---
 src/symboltable.rs | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index ca9594e7..cb53c017 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -221,11 +221,19 @@ impl<'a> SymbolTableAnalyzer<'a> {
         } else {
             match symbol.scope {
                 SymbolScope::Nonlocal => {
-                    // check if name is defined in parent table!
-                    let parent_symbol_table = self.tables.last();
-                    if let Some((symbols, _)) = parent_symbol_table {
-                        let scope_depth = self.tables.len();
-                        if !symbols.contains_key(&symbol.name) || scope_depth < 2 {
+                    let scope_depth = self.tables.len();
+                    if scope_depth > 0 {
+                        // check if the name is already defined in any outer scope
+                        // therefore
+                        if scope_depth < 2
+                            || !self
+                                .tables
+                                .iter()
+                                .skip(1) // omit the global scope as it is not non-local
+                                .rev() // revert the order for better performance
+                                .any(|t| t.0.contains_key(&symbol.name))
+                        // true when any of symbol tables contains the name -> then negate
+                        {
                             return Err(SymbolTableError {
                                 error: format!("no binding for nonlocal '{}' found", symbol.name),
                                 location: Default::default(),

From 6708cef2a4e4e3a513fb212cdff60c16edfe5931 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Tue, 2 Jun 2020 15:17:14 -0500
Subject: [PATCH 125/459] Fix compilation on redox

---
 src/compile.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index b2273bfa..b7e2f0a3 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -2042,7 +2042,7 @@ impl<O: OutputStream> Compiler<O> {
 
         let mut compile_element = |element| {
             self.compile_expression(element).map_err(|e| {
-                if matches!(e.error, CompileErrorType::InvalidStarExpr) {
+                if let CompileErrorType::InvalidStarExpr = e.error {
                     self.error(CompileErrorType::SyntaxError(
                         "iterable unpacking cannot be used in comprehension".to_owned(),
                     ))

From f7216a144ad3a8eed5387ce6fbf8976f4c22a532 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 6 Jun 2020 15:37:47 -0500
Subject: [PATCH 126/459] Fix Rust 1.44 clippy warnings

---
 src/compile.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index b7e2f0a3..7303e43e 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -591,7 +591,7 @@ impl<O: OutputStream> Compiler<O> {
                         {
                             return Err(self.error_loc(
                                 CompileErrorType::AsyncReturnValue,
-                                statement.location.clone(),
+                                statement.location,
                             ));
                         }
                         self.compile_expression(v)?;

From 71f260355679047d5c43c022789336020c12a5ff Mon Sep 17 00:00:00 2001
From: TheAnyKey <ich@simonroth.de>
Date: Sat, 13 Jun 2020 21:16:51 +0000
Subject: [PATCH 127/459] completion of error handling for named expressions
 and scope handling improvements

---
 src/symboltable.rs | 181 ++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 171 insertions(+), 10 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index cb53c017..2d71ea0d 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -105,6 +105,10 @@ pub struct Symbol {
     // indicates if the symbol gets a value assigned by a named expression in a comprehension
     // this is required to correct the scope in the analysis.
     pub is_assign_namedexpr_in_comprehension: bool,
+
+    // inidicates that the symbol is used a bound iterator variable. We distinguish this case
+    // from normal assignment to detect unallowed re-assignment to iterator variables.
+    pub is_iter: bool,
 }
 
 impl Symbol {
@@ -118,6 +122,7 @@ impl Symbol {
             is_parameter: false,
             is_free: false,
             is_assign_namedexpr_in_comprehension: false,
+            is_iter: false,
         }
     }
 
@@ -217,6 +222,10 @@ impl<'a> SymbolTableAnalyzer<'a> {
         if symbol.is_assign_namedexpr_in_comprehension
             && curr_st_typ == SymbolTableType::Comprehension
         {
+            // propagate symbol to next higher level that can hold it,
+            // i.e., function or module. Comprehension is skipped and
+            // Class is not allowed and detected as error.
+            //symbol.scope = SymbolScope::Nonlocal;
             self.analyze_symbol_comprehension(symbol, 0)?
         } else {
             match symbol.scope {
@@ -262,6 +271,74 @@ impl<'a> SymbolTableAnalyzer<'a> {
             }
         }
         Ok(())
+
+        /*let mut handled=false;
+        if curr_st_typ == SymbolTableType::Comprehension
+        {
+            if symbol.is_assign_namedexpr_in_comprehension {
+                // propagate symbol to next higher level that can hold it,
+                // i.e., function or module. Comprehension is skipped and
+                // Class is not allowed and detected as error.
+                //symbol.scope = SymbolScope::Nonlocal;
+                handled = true;
+                self.analyze_symbol_comprehension(symbol, 0)?
+            }
+            if symbol.is_iter {
+                if let Some(parent_symbol) = self.tables.last().unwrap().0.get(&symbol.name) {
+                    if parent_symbol.is_assigned {
+                        return Err(SymbolTableError {
+                            error: format!("assignment expression cannot rebind comprehension iteration variable {}", symbol.name),
+                            location: Default::default(),
+                        });
+                    }
+                }
+            }
+        }
+
+        if !handled {
+            match symbol.scope {
+                SymbolScope::Nonlocal => {
+                    let scope_depth = self.tables.len();
+                    if scope_depth > 0 {
+                        // check if the name is already defined in any outer scope
+                        // therefore
+                        if scope_depth < 2
+                            || !self
+                                .tables
+                                .iter()
+                                .skip(1) // omit the global scope as it is not non-local
+                                .rev() // revert the order for better performance
+                                .any(|t| t.0.contains_key(&symbol.name))
+                        // true when any of symbol tables contains the name -> then negate
+                        {
+                            return Err(SymbolTableError {
+                                error: format!("no binding for nonlocal '{}' found", symbol.name),
+                                location: Default::default(),
+                            });
+                        }
+                    } else {
+                        return Err(SymbolTableError {
+                            error: format!(
+                                "nonlocal {} defined at place without an enclosing scope",
+                                symbol.name
+                            ),
+                            location: Default::default(),
+                        });
+                    }
+                }
+                SymbolScope::Global => {
+                    // TODO: add more checks for globals?
+                }
+                SymbolScope::Local => {
+                    // all is well
+                }
+                SymbolScope::Unknown => {
+                    // Try hard to figure out what the scope of this symbol is.
+                    self.analyze_unknown_symbol(symbol);
+                }
+            }
+        }
+        Ok(())*/
     }
 
     fn analyze_unknown_symbol(&self, symbol: &mut Symbol) {
@@ -303,16 +380,35 @@ impl<'a> SymbolTableAnalyzer<'a> {
         let symbols = &mut last.0;
         let table_type = last.1;
 
+        // it is not allowed to use an iterator variable as assignee in a named expression
+        if
+        /*symbol.is_assign_namedexpr_in_comprehension && (maybe this could be asserted) */
+        symbol.is_iter {
+            return Err(SymbolTableError {
+                error: format!(
+                    "assignment expression cannot rebind comprehension iteration variable {}",
+                    symbol.name
+                ),
+                location: Default::default(),
+            });
+        }
+
         match table_type {
             SymbolTableType::Module => {
                 symbol.scope = SymbolScope::Global;
             }
-            SymbolTableType::Class => {}
+            SymbolTableType::Class => {
+                // named expressions are forbidden in comprehensions on class scope
+                return Err(SymbolTableError {
+                    error: "assignment expression within a comprehension cannot be used in a class body".to_string(),
+                    location: Default::default(),
+                } );
+            }
             SymbolTableType::Function => {
                 if let Some(parent_symbol) = symbols.get_mut(&symbol.name) {
                     if let SymbolScope::Unknown = parent_symbol.scope {
                         parent_symbol.is_assigned = true; // this information is new, as the asignment is done in inner scope
-                        self.analyze_unknown_symbol(symbol);
+                                                          //self.analyze_unknown_symbol(symbol); // not needed, symbol is analyzed anyhow when its scope is analyzed
                     }
 
                     match symbol.scope {
@@ -323,17 +419,34 @@ impl<'a> SymbolTableAnalyzer<'a> {
                             symbol.scope = SymbolScope::Nonlocal;
                         }
                     }
+                } else {
+                    let mut cloned_sym = symbol.clone();
+                    cloned_sym.scope = SymbolScope::Local;
+                    last.0.insert(cloned_sym.name.to_owned(), cloned_sym);
                 }
             }
             SymbolTableType::Comprehension => {
                 // TODO check for conflicts - requires more context information about variables
                 match symbols.get_mut(&symbol.name) {
                     Some(parent_symbol) => {
+                        // check if assignee is an iterator in top scope
+                        if parent_symbol.is_iter {
+                            return Err(SymbolTableError {
+                                error: format!("assignment expression cannot rebind comprehension iteration variable {}", symbol.name),
+                                location: Default::default(),
+                            });
+                        }
+
+                        // we synthesize the assignment to the symbol from inner scope
                         parent_symbol.is_assigned = true; // more checks are required
                     }
                     None => {
-                        let cloned_sym = symbol.clone();
-
+                        // extend the scope of the inner symbol
+                        // as we are in a nested comprehension, we expect that the symbol is needed
+                        // ouside, too, and set it therefore to non-local scope. I.e., we expect to
+                        // find a definition on a higher level
+                        let mut cloned_sym = symbol.clone();
+                        cloned_sym.scope = SymbolScope::Nonlocal;
                         last.0.insert(cloned_sym.name.to_owned(), cloned_sym);
                     }
                 }
@@ -353,6 +466,7 @@ enum SymbolUsage {
     Assigned,
     Parameter,
     AssignedNamedExprInCompr,
+    Iter,
 }
 
 #[derive(Default)]
@@ -369,6 +483,8 @@ enum ExpressionContext {
     Load,
     Store,
     Delete,
+    Iter,
+    IterDefinitionExp,
 }
 
 impl SymbolTableBuilder {
@@ -705,11 +821,14 @@ impl SymbolTableBuilder {
 
                 let mut is_first_generator = true;
                 for generator in generators {
-                    self.scan_expression(&generator.target, &ExpressionContext::Store)?;
+                    self.scan_expression(&generator.target, &ExpressionContext::Iter)?;
                     if is_first_generator {
                         is_first_generator = false;
                     } else {
-                        self.scan_expression(&generator.iter, &ExpressionContext::Load)?;
+                        self.scan_expression(
+                            &generator.iter,
+                            &ExpressionContext::IterDefinitionExp,
+                        )?;
                     }
 
                     for if_expr in &generator.ifs {
@@ -721,14 +840,22 @@ impl SymbolTableBuilder {
 
                 // The first iterable is passed as an argument into the created function:
                 assert!(!generators.is_empty());
-                self.scan_expression(&generators[0].iter, &ExpressionContext::Load)?;
+                self.scan_expression(&generators[0].iter, &ExpressionContext::IterDefinitionExp)?;
             }
             Call {
                 function,
                 args,
                 keywords,
             } => {
-                self.scan_expression(function, &ExpressionContext::Load)?;
+                match *context {
+                    ExpressionContext::IterDefinitionExp => {
+                        self.scan_expression(function, &ExpressionContext::IterDefinitionExp)?;
+                    }
+                    _ => {
+                        self.scan_expression(function, &ExpressionContext::Load)?;
+                    }
+                }
+
                 self.scan_expressions(args, &ExpressionContext::Load)?;
                 for keyword in keywords {
                     self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
@@ -743,17 +870,27 @@ impl SymbolTableBuilder {
                     ExpressionContext::Delete => {
                         self.register_name(name, SymbolUsage::Used)?;
                     }
-                    ExpressionContext::Load => {
+                    ExpressionContext::Load | ExpressionContext::IterDefinitionExp => {
                         self.register_name(name, SymbolUsage::Used)?;
                     }
                     ExpressionContext::Store => {
                         self.register_name(name, SymbolUsage::Assigned)?;
                     }
+                    ExpressionContext::Iter => {
+                        self.register_name(name, SymbolUsage::Iter)?;
+                    }
                 }
             }
             Lambda { args, body } => {
                 self.enter_function("lambda", args, expression.location.row())?;
-                self.scan_expression(body, &ExpressionContext::Load)?;
+                match *context {
+                    ExpressionContext::IterDefinitionExp => {
+                        self.scan_expression(body, &ExpressionContext::IterDefinitionExp)?;
+                    }
+                    _ => {
+                        self.scan_expression(body, &ExpressionContext::Load)?;
+                    }
+                }
                 self.leave_scope();
             }
             IfExpression { test, body, orelse } => {
@@ -763,6 +900,15 @@ impl SymbolTableBuilder {
             }
 
             NamedExpression { left, right } => {
+                // named expressions are not allowed in the definiton of
+                // comprehension iterator definitions
+                if let ExpressionContext::IterDefinitionExp = *context {
+                        return Err(SymbolTableError {
+                        error: "assignment expression cannot be used in a comprehension iterable expression".to_string(),
+                        location: Default::default(),
+                    });
+                }
+
                 self.scan_expression(right, &ExpressionContext::Load)?;
 
                 // special handling for assigned identifier in named expressions
@@ -933,8 +1079,23 @@ impl SymbolTableBuilder {
             SymbolUsage::Used => {
                 symbol.is_referenced = true;
             }
+            SymbolUsage::Iter => {
+                symbol.is_iter = true;
+            }
         }
 
+        // and even more checking
+        // it is not allowed to assign to iterator variables (by named expressions)
+        if symbol.is_iter && symbol.is_assigned
+        /*&& symbol.is_assign_namedexpr_in_comprehension*/
+        {
+            return Err(SymbolTableError {
+                error:
+                    "assignment expression cannot be used in a comprehension iterable expression"
+                        .to_string(),
+                location,
+            });
+        }
         Ok(())
     }
 }

From 162e5d02abe6fa99546a3a80639bdf55a89bc54c Mon Sep 17 00:00:00 2001
From: TheAnyKey <ich@simonroth.de>
Date: Sun, 14 Jun 2020 12:06:39 +0000
Subject: [PATCH 128/459] commented and cleaned up

---
 src/symboltable.rs | 74 ++--------------------------------------------
 1 file changed, 2 insertions(+), 72 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 2d71ea0d..e7bd6f0c 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -271,74 +271,6 @@ impl<'a> SymbolTableAnalyzer<'a> {
             }
         }
         Ok(())
-
-        /*let mut handled=false;
-        if curr_st_typ == SymbolTableType::Comprehension
-        {
-            if symbol.is_assign_namedexpr_in_comprehension {
-                // propagate symbol to next higher level that can hold it,
-                // i.e., function or module. Comprehension is skipped and
-                // Class is not allowed and detected as error.
-                //symbol.scope = SymbolScope::Nonlocal;
-                handled = true;
-                self.analyze_symbol_comprehension(symbol, 0)?
-            }
-            if symbol.is_iter {
-                if let Some(parent_symbol) = self.tables.last().unwrap().0.get(&symbol.name) {
-                    if parent_symbol.is_assigned {
-                        return Err(SymbolTableError {
-                            error: format!("assignment expression cannot rebind comprehension iteration variable {}", symbol.name),
-                            location: Default::default(),
-                        });
-                    }
-                }
-            }
-        }
-
-        if !handled {
-            match symbol.scope {
-                SymbolScope::Nonlocal => {
-                    let scope_depth = self.tables.len();
-                    if scope_depth > 0 {
-                        // check if the name is already defined in any outer scope
-                        // therefore
-                        if scope_depth < 2
-                            || !self
-                                .tables
-                                .iter()
-                                .skip(1) // omit the global scope as it is not non-local
-                                .rev() // revert the order for better performance
-                                .any(|t| t.0.contains_key(&symbol.name))
-                        // true when any of symbol tables contains the name -> then negate
-                        {
-                            return Err(SymbolTableError {
-                                error: format!("no binding for nonlocal '{}' found", symbol.name),
-                                location: Default::default(),
-                            });
-                        }
-                    } else {
-                        return Err(SymbolTableError {
-                            error: format!(
-                                "nonlocal {} defined at place without an enclosing scope",
-                                symbol.name
-                            ),
-                            location: Default::default(),
-                        });
-                    }
-                }
-                SymbolScope::Global => {
-                    // TODO: add more checks for globals?
-                }
-                SymbolScope::Local => {
-                    // all is well
-                }
-                SymbolScope::Unknown => {
-                    // Try hard to figure out what the scope of this symbol is.
-                    self.analyze_unknown_symbol(symbol);
-                }
-            }
-        }
-        Ok(())*/
     }
 
     fn analyze_unknown_symbol(&self, symbol: &mut Symbol) {
@@ -381,9 +313,7 @@ impl<'a> SymbolTableAnalyzer<'a> {
         let table_type = last.1;
 
         // it is not allowed to use an iterator variable as assignee in a named expression
-        if
-        /*symbol.is_assign_namedexpr_in_comprehension && (maybe this could be asserted) */
-        symbol.is_iter {
+        if symbol.is_iter {
             return Err(SymbolTableError {
                 error: format!(
                     "assignment expression cannot rebind comprehension iteration variable {}",
@@ -903,7 +833,7 @@ impl SymbolTableBuilder {
                 // named expressions are not allowed in the definiton of
                 // comprehension iterator definitions
                 if let ExpressionContext::IterDefinitionExp = *context {
-                        return Err(SymbolTableError {
+                    return Err(SymbolTableError {
                         error: "assignment expression cannot be used in a comprehension iterable expression".to_string(),
                         location: Default::default(),
                     });

From 60a787e3e058ddfeb43e6b3cc202c482a7c80eaf Mon Sep 17 00:00:00 2001
From: TheAnyKey <ich@simonroth.de>
Date: Sun, 21 Jun 2020 22:18:43 +0000
Subject: [PATCH 129/459] fixed scoping issue, commented test with references
 to original version

---
 src/symboltable.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index e7bd6f0c..58c27eb4 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -1011,6 +1011,7 @@ impl SymbolTableBuilder {
             }
             SymbolUsage::Iter => {
                 symbol.is_iter = true;
+                symbol.scope = SymbolScope::Local;
             }
         }
 

From 40576c7a840dc0b2c731be568186cccf0e085a17 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Tue, 23 Jun 2020 18:47:08 -0500
Subject: [PATCH 130/459] Release 0.1.2

rustpython@0.1.2
rustpython-bytecode@0.1.2
rustpython-compiler@0.1.2
rustpython-derive@0.1.2
rustpython-parser@0.1.2
rustpython-vm@0.1.2
rustpython_freeze@0.1.2
rustpython_wasm@0.1.2

Generated by cargo-workspaces
---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 54eca247..d968cb11 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "rustpython-compiler"
-version = "0.1.1"
+version = "0.1.2"
 description = "Compiler for python code into bytecode for the rustpython VM."
 authors = ["RustPython Team"]
 repository = "https://github.com/RustPython/RustPython"

From a3b7a837417aeebee2bc892a6a2bc9403368a4cc Mon Sep 17 00:00:00 2001
From: lynskylate <lynskylate@gmail.com>
Date: Tue, 7 Jul 2020 06:00:50 +0800
Subject: [PATCH 131/459] Add annotation setup

---
 src/compile.rs | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/src/compile.rs b/src/compile.rs
index 7303e43e..ceb3743d 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -984,6 +984,64 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
+    fn option_stmt_to_bool(&mut self, stmts: &Option<ast::Suite>) -> bool {
+        match &stmts {
+            Some(stmts) => self.find_ann(stmts),
+            None => false,
+        }
+    }
+
+    fn find_ann(&mut self, body: &[ast::Statement]) -> bool {
+        use ast::StatementType::*;
+
+        for statement in body {
+            let res = match &statement.node {
+                AnnAssign {
+                    target: _,
+                    annotation: _,
+                    value: _,
+                } => true,
+                For {
+                    is_async: _,
+                    target: _,
+                    iter: _,
+                    body,
+                    orelse,
+                } => self.find_ann(body) || self.option_stmt_to_bool(orelse),
+                If {
+                    test: _,
+                    body,
+                    orelse,
+                } => self.find_ann(body) || self.option_stmt_to_bool(orelse),
+                While {
+                    test: _,
+                    body,
+                    orelse,
+                } => self.find_ann(body) || self.option_stmt_to_bool(orelse),
+                With {
+                    is_async: _,
+                    items: _,
+                    body,
+                } => self.find_ann(body),
+                Try {
+                    body,
+                    handlers: _,
+                    orelse,
+                    finalbody,
+                } => {
+                    self.find_ann(body)
+                        || self.option_stmt_to_bool(orelse)
+                        || self.option_stmt_to_bool(finalbody)
+                }
+                _ => false,
+            };
+            if res {
+                return true;
+            }
+        }
+        return false;
+    }
+
     fn compile_class_def(
         &mut self,
         name: &str,
@@ -1037,6 +1095,18 @@ impl<O: OutputStream> Compiler<O> {
             name: "__qualname__".to_owned(),
             scope: bytecode::NameScope::Free,
         });
+        // setup annotations
+        if self.find_ann(body) {
+            self.emit(Instruction::BuildMap {
+                size: 0,
+                unpack: false,
+                for_call: false,
+            });
+            self.emit(Instruction::StoreName {
+                name: "__annotations__".to_owned(),
+                scope: bytecode::NameScope::Free,
+            });
+        }
         self.compile_statements(new_body)?;
         self.emit(Instruction::LoadConst {
             value: bytecode::Constant::None,

From e1999cb6982c0284f2741768f9c05a9cc4d214ad Mon Sep 17 00:00:00 2001
From: lynskylate <lynskylate@gmail.com>
Date: Tue, 7 Jul 2020 06:11:02 +0800
Subject: [PATCH 132/459] Remove needless return

---
 src/compile.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index ceb3743d..af3d676d 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1039,7 +1039,7 @@ impl<O: OutputStream> Compiler<O> {
                 return true;
             }
         }
-        return false;
+        false
     }
 
     fn compile_class_def(

From fae42a053309b90388f5b61bd514a40371080122 Mon Sep 17 00:00:00 2001
From: lynskylate <lynskylate@gmail.com>
Date: Wed, 8 Jul 2020 01:06:12 +0800
Subject: [PATCH 133/459] Add annotation setup

---
 src/compile.rs | 37 ++++++++++++++-----------------------
 1 file changed, 14 insertions(+), 23 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index af3d676d..ee81ff7d 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -984,15 +984,14 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
-    fn option_stmt_to_bool(&mut self, stmts: &Option<ast::Suite>) -> bool {
-        match &stmts {
-            Some(stmts) => self.find_ann(stmts),
-            None => false,
-        }
-    }
-
-    fn find_ann(&mut self, body: &[ast::Statement]) -> bool {
+    fn find_ann(&self, body: &[ast::Statement]) -> bool {
         use ast::StatementType::*;
+        let option_stmt_to_bool = |suit: &Option<ast::Suite>| -> bool {
+            match suit {
+                Some(stmts) => self.find_ann(stmts),
+                None => false,
+            }
+        };
 
         for statement in body {
             let res = match &statement.node {
@@ -1007,17 +1006,17 @@ impl<O: OutputStream> Compiler<O> {
                     iter: _,
                     body,
                     orelse,
-                } => self.find_ann(body) || self.option_stmt_to_bool(orelse),
+                } => self.find_ann(body) || option_stmt_to_bool(orelse),
                 If {
                     test: _,
                     body,
                     orelse,
-                } => self.find_ann(body) || self.option_stmt_to_bool(orelse),
+                } => self.find_ann(body) || option_stmt_to_bool(orelse),
                 While {
                     test: _,
                     body,
                     orelse,
-                } => self.find_ann(body) || self.option_stmt_to_bool(orelse),
+                } => self.find_ann(body) || option_stmt_to_bool(orelse),
                 With {
                     is_async: _,
                     items: _,
@@ -1029,9 +1028,9 @@ impl<O: OutputStream> Compiler<O> {
                     orelse,
                     finalbody,
                 } => {
-                    self.find_ann(body)
-                        || self.option_stmt_to_bool(orelse)
-                        || self.option_stmt_to_bool(finalbody)
+                    self.find_ann(&body)
+                        || option_stmt_to_bool(orelse)
+                        || option_stmt_to_bool(finalbody)
                 }
                 _ => false,
             };
@@ -1097,15 +1096,7 @@ impl<O: OutputStream> Compiler<O> {
         });
         // setup annotations
         if self.find_ann(body) {
-            self.emit(Instruction::BuildMap {
-                size: 0,
-                unpack: false,
-                for_call: false,
-            });
-            self.emit(Instruction::StoreName {
-                name: "__annotations__".to_owned(),
-                scope: bytecode::NameScope::Free,
-            });
+            self.emit(Instruction::SetupAnnotation);
         }
         self.compile_statements(new_body)?;
         self.emit(Instruction::LoadConst {

From 1125a529e7951844be45074a1da21e0198dd67c1 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Sun, 12 Jul 2020 21:33:11 +0900
Subject: [PATCH 134/459] CompilerError always contains source_path

---
 src/compile.rs     | 49 +++++++++++++++++++++-------------------------
 src/error.rs       | 27 ++++++++++---------------
 src/symboltable.rs |  6 +++---
 3 files changed, 35 insertions(+), 47 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index ee81ff7d..41710c85 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -26,7 +26,7 @@ struct Compiler<O: OutputStream = BasicOutputStream> {
     output_stack: Vec<O>,
     symbol_table_stack: Vec<SymbolTable>,
     nxt_label: usize,
-    source_path: Option<String>,
+    source_path: String,
     current_source_location: ast::Location,
     current_qualified_path: Option<String>,
     done_with_future_stmts: bool,
@@ -75,17 +75,19 @@ pub fn compile(
     source_path: String,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
+    let to_compile_error =
+        |parse_error| CompileError::from_parse_error(parse_error, source_path.clone());
     match mode {
         Mode::Exec => {
-            let ast = parser::parse_program(source)?;
+            let ast = parser::parse_program(source).map_err(to_compile_error)?;
             compile_program(ast, source_path, opts)
         }
         Mode::Eval => {
-            let statement = parser::parse_statement(source)?;
+            let statement = parser::parse_statement(source).map_err(to_compile_error)?;
             compile_statement_eval(statement, source_path, opts)
         }
         Mode::Single => {
-            let ast = parser::parse_program(source)?;
+            let ast = parser::parse_program(source).map_err(to_compile_error)?;
             compile_program_single(ast, source_path, opts)
         }
     }
@@ -97,8 +99,7 @@ fn with_compiler(
     opts: CompileOpts,
     f: impl FnOnce(&mut Compiler) -> CompileResult<()>,
 ) -> CompileResult<CodeObject> {
-    let mut compiler = Compiler::new(opts);
-    compiler.source_path = Some(source_path);
+    let mut compiler = Compiler::new(opts, source_path);
     compiler.push_new_code_object("<module>".to_owned());
     f(&mut compiler)?;
     let code = compiler.pop_code_object();
@@ -112,8 +113,9 @@ pub fn compile_program(
     source_path: String,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
+    let symbol_table = make_symbol_table(&ast)
+        .map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?;
     with_compiler(source_path, opts, |compiler| {
-        let symbol_table = make_symbol_table(&ast)?;
         compiler.compile_program(&ast, symbol_table)
     })
 }
@@ -124,8 +126,9 @@ pub fn compile_statement_eval(
     source_path: String,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
+    let symbol_table = statements_to_symbol_table(&statement)
+        .map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?;
     with_compiler(source_path, opts, |compiler| {
-        let symbol_table = statements_to_symbol_table(&statement)?;
         compiler.compile_statement_eval(&statement, symbol_table)
     })
 }
@@ -136,28 +139,20 @@ pub fn compile_program_single(
     source_path: String,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
+    let symbol_table = make_symbol_table(&ast)
+        .map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?;
     with_compiler(source_path, opts, |compiler| {
-        let symbol_table = make_symbol_table(&ast)?;
         compiler.compile_program_single(&ast, symbol_table)
     })
 }
 
-impl<O> Default for Compiler<O>
-where
-    O: OutputStream,
-{
-    fn default() -> Self {
-        Compiler::new(CompileOpts::default())
-    }
-}
-
 impl<O: OutputStream> Compiler<O> {
-    fn new(opts: CompileOpts) -> Self {
+    fn new(opts: CompileOpts, source_path: String) -> Self {
         Compiler {
             output_stack: Vec::new(),
             symbol_table_stack: Vec::new(),
             nxt_label: 0,
-            source_path: None,
+            source_path,
             current_source_location: ast::Location::default(),
             current_qualified_path: None,
             done_with_future_stmts: false,
@@ -194,7 +189,7 @@ impl<O: OutputStream> Compiler<O> {
             None,
             Vec::new(),
             None,
-            self.source_path.clone().unwrap(),
+            self.source_path.clone(),
             line_number,
             obj_name,
         ));
@@ -733,7 +728,7 @@ impl<O: OutputStream> Compiler<O> {
             varargs_name,
             args.kwonlyargs.iter().map(|a| a.arg.clone()).collect(),
             varkeywords_name,
-            self.source_path.clone().unwrap(),
+            self.source_path.clone(),
             line_number,
             name.to_owned(),
         ));
@@ -1069,7 +1064,7 @@ impl<O: OutputStream> Compiler<O> {
             None,
             vec![],
             None,
-            self.source_path.clone().unwrap(),
+            self.source_path.clone(),
             line_number,
             name.to_owned(),
         ));
@@ -2032,7 +2027,7 @@ impl<O: OutputStream> Compiler<O> {
             None,
             vec![],
             None,
-            self.source_path.clone().unwrap(),
+            self.source_path.clone(),
             line_number,
             name.clone(),
         ));
@@ -2377,7 +2372,7 @@ fn compile_conversion_flag(conversion_flag: ast::ConversionFlag) -> bytecode::Co
 
 #[cfg(test)]
 mod tests {
-    use super::Compiler;
+    use super::{CompileOpts, Compiler};
     use crate::symboltable::make_symbol_table;
     use rustpython_bytecode::bytecode::Constant::*;
     use rustpython_bytecode::bytecode::Instruction::*;
@@ -2385,8 +2380,8 @@ mod tests {
     use rustpython_parser::parser;
 
     fn compile_exec(source: &str) -> CodeObject {
-        let mut compiler: Compiler = Default::default();
-        compiler.source_path = Some("source_path".to_owned());
+        let mut compiler: Compiler =
+            Compiler::new(CompileOpts::default(), "source_path".to_owned());
         compiler.push_new_code_object("<module>".to_owned());
         let ast = parser::parse_program(source).unwrap();
         let symbol_scope = make_symbol_table(&ast).unwrap();
diff --git a/src/error.rs b/src/error.rs
index 86635bc9..35a7828f 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -10,29 +10,22 @@ pub struct CompileError {
     pub statement: Option<String>,
     pub error: CompileErrorType,
     pub location: Location,
-    pub source_path: Option<String>,
+    pub source_path: String,
 }
 
 impl CompileError {
-    pub fn update_statement_info(&mut self, statement: String) {
-        self.statement = Some(statement);
-    }
-
-    pub fn update_source_path(&mut self, source_path: &str) {
-        debug_assert!(self.source_path.is_none());
-        self.source_path = Some(source_path.to_owned());
-    }
-}
-
-impl From<ParseError> for CompileError {
-    fn from(error: ParseError) -> Self {
-        CompileError {
+    pub fn from_parse_error(parse_error: ParseError, source_path: String) -> Self {
+        Self {
             statement: None,
-            error: CompileErrorType::Parse(error.error),
-            location: error.location,
-            source_path: None,
+            error: CompileErrorType::Parse(parse_error.error),
+            location: parse_error.location,
+            source_path,
         }
     }
+
+    pub fn update_statement_info(&mut self, statement: String) {
+        self.statement = Some(statement);
+    }
 }
 
 #[derive(Debug)]
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 58c27eb4..5714367b 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -149,13 +149,13 @@ pub struct SymbolTableError {
     location: Location,
 }
 
-impl From<SymbolTableError> for CompileError {
-    fn from(error: SymbolTableError) -> Self {
+impl CompileError {
+    pub fn from_symbol_table_error(error: SymbolTableError, source_path: String) -> Self {
         CompileError {
             statement: None,
             error: CompileErrorType::SyntaxError(error.error),
             location: error.location,
-            source_path: None,
+            source_path,
         }
     }
 }

From f6e25fc0cb5757c449e18ced85602adb510e6643 Mon Sep 17 00:00:00 2001
From: Yiqun Ling <lynskylate@gmail.com>
Date: Tue, 14 Jul 2020 17:58:58 +0800
Subject: [PATCH 135/459] Set annotation for function args

---
 src/compile.rs | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index 41710c85..66d3081e 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -931,7 +931,7 @@ impl<O: OutputStream> Compiler<O> {
             num_annotations += 1;
         }
 
-        for arg in args.args.iter() {
+        let mut visit_arg_annotation = |arg: &ast::Parameter| -> CompileResult<()> {
             if let Some(annotation) = &arg.annotation {
                 self.emit(Instruction::LoadConst {
                     value: bytecode::Constant::String {
@@ -941,6 +941,19 @@ impl<O: OutputStream> Compiler<O> {
                 self.compile_expression(&annotation)?;
                 num_annotations += 1;
             }
+            Ok(())
+        };
+
+        for arg in args.args.iter().chain(args.kwonlyargs.iter()) {
+            visit_arg_annotation(arg)?;
+        }
+
+        if let ast::Varargs::Named(arg) = &args.vararg {
+            visit_arg_annotation(arg)?;
+        }
+
+        if let ast::Varargs::Named(arg) = &args.kwarg {
+            visit_arg_annotation(arg)?;
         }
 
         if num_annotations > 0 {
@@ -1385,6 +1398,11 @@ impl<O: OutputStream> Compiler<O> {
             self.compile_store(target)?;
         }
 
+        // Annotations are only evaluated in a module or class.
+        if self.ctx.in_func() {
+            return Ok(());
+        }
+
         // Compile annotation:
         self.compile_expression(annotation)?;
 

From 5ca3fc79f47dd1040b11467cd96391a8f0606e4a Mon Sep 17 00:00:00 2001
From: lynskylate <lynskylate@gmail.com>
Date: Sat, 18 Jul 2020 04:02:42 +0800
Subject: [PATCH 136/459] Try to use loadname local replace loadname free

---
 src/compile.rs     | 2 +-
 src/symboltable.rs | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 66d3081e..69adf4d9 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -298,7 +298,7 @@ impl<O: OutputStream> Compiler<O> {
             SymbolScope::Global => bytecode::NameScope::Global,
             SymbolScope::Nonlocal => bytecode::NameScope::NonLocal,
             SymbolScope::Unknown => bytecode::NameScope::Free,
-            SymbolScope::Local => bytecode::NameScope::Free,
+            SymbolScope::Local => bytecode::NameScope::Local,
         }
     }
 
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 5714367b..1b9707da 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -276,6 +276,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
     fn analyze_unknown_symbol(&self, symbol: &mut Symbol) {
         if symbol.is_assigned || symbol.is_parameter {
             symbol.scope = SymbolScope::Local;
+        } else if symbol.is_referenced {
+            symbol.scope = SymbolScope::Unknown;
         } else {
             // Interesting stuff about the __class__ variable:
             // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
@@ -680,11 +682,11 @@ impl SymbolTableBuilder {
                 self.scan_expressions(vals, context)?;
             }
             Subscript { a, b } => {
-                self.scan_expression(a, context)?;
+                self.scan_expression(a, &ExpressionContext::Load)?;
                 self.scan_expression(b, context)?;
             }
             Attribute { value, .. } => {
-                self.scan_expression(value, context)?;
+                self.scan_expression(value, &ExpressionContext::Load)?;
             }
             Dict { elements } => {
                 for (key, value) in elements {

From dc4acdf19e740aa9fd9ba4857a8e618a1643db32 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Tue, 21 Jul 2020 00:47:22 -0500
Subject: [PATCH 137/459] Update itertools

---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index d968cb11..6846c438 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,7 +9,7 @@ edition = "2018"
 
 [dependencies]
 indexmap = "1.0"
-itertools = "0.8.0"
+itertools = "0.9"
 rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
 rustpython-parser = { path = "../parser", version = "0.1.1" }
 num-complex = { version = "0.2", features = ["serde"] }

From 66731121ab3af0f44e290e2ffc25613267dcc076 Mon Sep 17 00:00:00 2001
From: lynskylate <lynskylate@gmail.com>
Date: Fri, 31 Jul 2020 02:11:00 +0800
Subject: [PATCH 138/459] Introduce symtable test and add is_annotation for
 symbol

---
 src/symboltable.rs | 42 ++++++++++++++++++++++++++++++++++++++----
 1 file changed, 38 insertions(+), 4 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 5714367b..f2f0aabd 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -97,10 +97,13 @@ pub struct Symbol {
     pub name: String,
     // pub table: SymbolTableRef,
     pub scope: SymbolScope,
+    // TODO(lynskylate@gmail.com): Use bitflags replace
     pub is_referenced: bool,
     pub is_assigned: bool,
     pub is_parameter: bool,
     pub is_free: bool,
+    pub is_annotated: bool,
+    pub is_imported: bool,
 
     // indicates if the symbol gets a value assigned by a named expression in a comprehension
     // this is required to correct the scope in the analysis.
@@ -121,6 +124,8 @@ impl Symbol {
             is_assigned: false,
             is_parameter: false,
             is_free: false,
+            is_annotated: false,
+            is_imported: false,
             is_assign_namedexpr_in_comprehension: false,
             is_iter: false,
         }
@@ -276,6 +281,11 @@ impl<'a> SymbolTableAnalyzer<'a> {
     fn analyze_unknown_symbol(&self, symbol: &mut Symbol) {
         if symbol.is_assigned || symbol.is_parameter {
             symbol.scope = SymbolScope::Local;
+        } else if !symbol.is_assigned && symbol.is_referenced {
+            // if symbol is referenced in its block, but not assigned to.
+            // https://docs.python.org/3/library/symtable.html?highlight=symtable#symtable.Symbol.is_free
+            symbol.scope = SymbolScope::Unknown;
+            symbol.is_free = true;
         } else {
             // Interesting stuff about the __class__ variable:
             // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
@@ -394,7 +404,10 @@ enum SymbolUsage {
     Nonlocal,
     Used,
     Assigned,
+    Imported,
+    AnnotationAssigned,
     Parameter,
+    AnnotationParameter,
     AssignedNamedExprInCompr,
     Iter,
 }
@@ -460,7 +473,11 @@ impl SymbolTableBuilder {
     }
 
     fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
-        self.register_name(&parameter.arg, SymbolUsage::Parameter)
+        if let Some(_annotation) = &parameter.annotation {
+            self.register_name(&parameter.arg, SymbolUsage::AnnotationParameter)
+        } else {
+            self.register_name(&parameter.arg, SymbolUsage::Parameter)
+        }
     }
 
     fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
@@ -564,12 +581,12 @@ impl SymbolTableBuilder {
                 for name in names {
                     if let Some(alias) = &name.alias {
                         // `import mymodule as myalias`
-                        self.register_name(alias, SymbolUsage::Assigned)?;
+                        self.register_name(alias, SymbolUsage::Imported)?;
                     } else {
                         // `import module`
                         self.register_name(
                             name.symbol.split('.').next().unwrap(),
-                            SymbolUsage::Assigned,
+                            SymbolUsage::Imported,
                         )?;
                     }
                 }
@@ -601,7 +618,12 @@ impl SymbolTableBuilder {
                 annotation,
                 value,
             } => {
-                self.scan_expression(target, &ExpressionContext::Store)?;
+                // https://github.com/python/cpython/blob/master/Python/symtable.c#L1233
+                if let ast::ExpressionType::Identifier { ref name } = target.node {
+                    self.register_name(name, SymbolUsage::AnnotationAssigned)?;
+                } else {
+                    self.scan_expression(target, &ExpressionContext::Store)?;
+                }
                 self.scan_expression(annotation, &ExpressionContext::Load)?;
                 if let Some(value) = value {
                     self.scan_expression(value, &ExpressionContext::Load)?;
@@ -984,9 +1006,21 @@ impl SymbolTableBuilder {
                     });
                 }
             }
+            SymbolUsage::Imported => {
+                symbol.is_assigned = true;
+                symbol.is_imported = true;
+            }
             SymbolUsage::Parameter => {
                 symbol.is_parameter = true;
             }
+            SymbolUsage::AnnotationParameter => {
+                symbol.is_parameter = true;
+                symbol.is_annotated = true;
+            }
+            SymbolUsage::AnnotationAssigned => {
+                symbol.is_assigned = true;
+                symbol.is_annotated = true;
+            }
             SymbolUsage::Assigned => {
                 symbol.is_assigned = true;
             }

From 70eeb2874891d0d8efdae454dc497e5c7027cccf Mon Sep 17 00:00:00 2001
From: Yiqun Ling <lynskylate@gmail.com>
Date: Sat, 1 Aug 2020 01:29:11 +0800
Subject: [PATCH 139/459] Apply suggestions from code review

Co-authored-by: Jeong YunWon <youknowone@users.noreply.github.com>
---
 src/symboltable.rs | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index f2f0aabd..811fd48b 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -473,11 +473,12 @@ impl SymbolTableBuilder {
     }
 
     fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
-        if let Some(_annotation) = &parameter.annotation {
-            self.register_name(&parameter.arg, SymbolUsage::AnnotationParameter)
+        let usage = if parameter.annotation.is_some() {
+            SymbolUsage::AnnotationParamater
         } else {
-            self.register_name(&parameter.arg, SymbolUsage::Parameter)
-        }
+            SymbolUsage::Parameter
+        };
+        self.regsiter_name(&parameter.arg, usage)
     }
 
     fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {

From dcca6624329b991889d8f82497dcb224c578c131 Mon Sep 17 00:00:00 2001
From: lynskylate <lynskylate@gmail.com>
Date: Sat, 1 Aug 2020 02:05:44 +0800
Subject: [PATCH 140/459] Optimize code

---
 src/symboltable.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 811fd48b..79e0b2be 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -97,7 +97,7 @@ pub struct Symbol {
     pub name: String,
     // pub table: SymbolTableRef,
     pub scope: SymbolScope,
-    // TODO(lynskylate@gmail.com): Use bitflags replace
+    // TODO: Use bitflags replace
     pub is_referenced: bool,
     pub is_assigned: bool,
     pub is_parameter: bool,
@@ -474,11 +474,11 @@ impl SymbolTableBuilder {
 
     fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
         let usage = if parameter.annotation.is_some() {
-            SymbolUsage::AnnotationParamater
+            SymbolUsage::AnnotationParameter
         } else {
             SymbolUsage::Parameter
         };
-        self.regsiter_name(&parameter.arg, usage)
+        self.register_name(&parameter.arg, usage)
     }
 
     fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {

From 32232811fb57febcbada51b7effaea0441a4d9d5 Mon Sep 17 00:00:00 2001
From: lynskylate <lynskylate@gmail.com>
Date: Sun, 2 Aug 2020 03:43:19 +0800
Subject: [PATCH 141/459] Fix scope change

---
 src/symboltable.rs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 79e0b2be..e8b80f09 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -281,11 +281,6 @@ impl<'a> SymbolTableAnalyzer<'a> {
     fn analyze_unknown_symbol(&self, symbol: &mut Symbol) {
         if symbol.is_assigned || symbol.is_parameter {
             symbol.scope = SymbolScope::Local;
-        } else if !symbol.is_assigned && symbol.is_referenced {
-            // if symbol is referenced in its block, but not assigned to.
-            // https://docs.python.org/3/library/symtable.html?highlight=symtable#symtable.Symbol.is_free
-            symbol.scope = SymbolScope::Unknown;
-            symbol.is_free = true;
         } else {
             // Interesting stuff about the __class__ variable:
             // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object

From f6eb1168bc180415ed68314527d7468a2ddd1b8f Mon Sep 17 00:00:00 2001
From: lynskylate <lynskylate@gmail.com>
Date: Sat, 8 Aug 2020 01:53:13 +0800
Subject: [PATCH 142/459] Fix scope error

---
 src/compile.rs     | 10 +++++++++-
 src/symboltable.rs |  2 --
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 69adf4d9..26eca75a 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -298,7 +298,15 @@ impl<O: OutputStream> Compiler<O> {
             SymbolScope::Global => bytecode::NameScope::Global,
             SymbolScope::Nonlocal => bytecode::NameScope::NonLocal,
             SymbolScope::Unknown => bytecode::NameScope::Free,
-            SymbolScope::Local => bytecode::NameScope::Local,
+            SymbolScope::Local => {
+                // Only in function block, we use load local
+                // https://github.com/python/cpython/blob/master/Python/compile.c#L3582
+                if self.ctx.in_func() {
+                    bytecode::NameScope::Local
+                } else {
+                    bytecode::NameScope::Free
+                }
+            }
         }
     }
 
diff --git a/src/symboltable.rs b/src/symboltable.rs
index d121fa19..fd234270 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -281,8 +281,6 @@ impl<'a> SymbolTableAnalyzer<'a> {
     fn analyze_unknown_symbol(&self, symbol: &mut Symbol) {
         if symbol.is_assigned || symbol.is_parameter {
             symbol.scope = SymbolScope::Local;
-        } else if symbol.is_referenced {
-            symbol.scope = SymbolScope::Unknown;
         } else {
             // Interesting stuff about the __class__ variable:
             // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object

From 2b54fa871ff739027ca06165754451a58429721f Mon Sep 17 00:00:00 2001
From: lynskylate <lynskylate@gmail.com>
Date: Sat, 8 Aug 2020 13:08:13 +0800
Subject: [PATCH 143/459] Fix load context

---
 src/symboltable.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index fd234270..75a8c069 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -699,7 +699,7 @@ impl SymbolTableBuilder {
             }
             Subscript { a, b } => {
                 self.scan_expression(a, &ExpressionContext::Load)?;
-                self.scan_expression(b, context)?;
+                self.scan_expression(b, &ExpressionContext::Load)?;
             }
             Attribute { value, .. } => {
                 self.scan_expression(value, &ExpressionContext::Load)?;

From 7714871b37102cacce048d71f1993cf0ce7a98fc Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Sun, 9 Aug 2020 21:33:13 +0900
Subject: [PATCH 144/459] clean up PyComplex

---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 6846c438..9c73674e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -12,6 +12,6 @@ indexmap = "1.0"
 itertools = "0.9"
 rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
 rustpython-parser = { path = "../parser", version = "0.1.1" }
-num-complex = { version = "0.2", features = ["serde"] }
+num-complex = { version = "0.3", features = ["serde"] }
 log = "0.4"
 arrayvec = "0.5"

From d8c8b11726628804f57ebff49e3f5b919a9eab9d Mon Sep 17 00:00:00 2001
From: Yiqun Ling <lynskylate@gmail.com>
Date: Thu, 13 Aug 2020 03:12:28 +0800
Subject: [PATCH 145/459] Add nested and optimized for symtable

---
 src/symboltable.rs | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 75a8c069..c3662632 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -41,6 +41,9 @@ pub struct SymbolTable {
     /// The line number in the sourcecode where this symboltable begins.
     pub line_number: usize,
 
+    // Return True if the block is a nested class or function
+    pub is_nested: bool,
+
     /// A set of symbols present on this scope level.
     pub symbols: IndexMap<String, Symbol>,
 
@@ -50,11 +53,12 @@ pub struct SymbolTable {
 }
 
 impl SymbolTable {
-    fn new(name: String, typ: SymbolTableType, line_number: usize) -> Self {
+    fn new(name: String, typ: SymbolTableType, line_number: usize, is_nested: bool) -> Self {
         SymbolTable {
             name,
             typ,
             line_number,
+            is_nested,
             symbols: Default::default(),
             sub_tables: vec![],
         }
@@ -438,7 +442,12 @@ impl SymbolTableBuilder {
     }
 
     fn enter_scope(&mut self, name: &str, typ: SymbolTableType, line_number: usize) {
-        let table = SymbolTable::new(name.to_owned(), typ, line_number);
+        let is_nested = self
+            .tables
+            .last()
+            .map(|table| table.is_nested || table.typ == SymbolTableType::Function)
+            .unwrap_or(false);
+        let table = SymbolTable::new(name.to_owned(), typ, line_number, is_nested);
         self.tables.push(table);
     }
 

From 7f0798b235c830f2eb93fce36c984d717ff56bc9 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 12 Aug 2020 16:03:20 -0500
Subject: [PATCH 146/459] Fix some minor things required for unittest

---
 src/compile.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index 26eca75a..a3b75844 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -479,7 +479,9 @@ impl<O: OutputStream> Compiler<O> {
 
                 self.compile_statements(body)?;
 
-                for end_label in end_labels {
+                // sort of "stack up" the layers of with blocks:
+                // with a, b: body -> start_with(a) start_with(b) body() end_with(b) end_with(a)
+                for end_label in end_labels.into_iter().rev() {
                     self.emit(Instruction::PopBlock);
                     self.emit(Instruction::EnterFinally);
                     self.set_label(end_label);

From f9fe1c0423272df4aab9fbad3a874c0fdd36f0d2 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Thu, 20 Aug 2020 04:01:25 +0900
Subject: [PATCH 147/459] Revert "Merge pull request #2106 from
 RustPython/coolreader18/unittest-tests"

This reverts commit 96926daf6ffa2a8dbb374b3c6169b378bd0d67ef, reversing
changes made to 230f91582f98ff74abeb8759718e49c386e2b398.
---
 src/compile.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index a3b75844..26eca75a 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -479,9 +479,7 @@ impl<O: OutputStream> Compiler<O> {
 
                 self.compile_statements(body)?;
 
-                // sort of "stack up" the layers of with blocks:
-                // with a, b: body -> start_with(a) start_with(b) body() end_with(b) end_with(a)
-                for end_label in end_labels.into_iter().rev() {
+                for end_label in end_labels {
                     self.emit(Instruction::PopBlock);
                     self.emit(Instruction::EnterFinally);
                     self.set_label(end_label);

From a251109090a2a9fa8650fe1983b4289b975a6050 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 12 Aug 2020 16:03:20 -0500
Subject: [PATCH 148/459] Fix some minor things required for unittest

---
 src/compile.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index 26eca75a..a3b75844 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -479,7 +479,9 @@ impl<O: OutputStream> Compiler<O> {
 
                 self.compile_statements(body)?;
 
-                for end_label in end_labels {
+                // sort of "stack up" the layers of with blocks:
+                // with a, b: body -> start_with(a) start_with(b) body() end_with(b) end_with(a)
+                for end_label in end_labels.into_iter().rev() {
                     self.emit(Instruction::PopBlock);
                     self.emit(Instruction::EnterFinally);
                     self.set_label(end_label);

From de7396347f96f2f7ef433c1e69c760051e52ff72 Mon Sep 17 00:00:00 2001
From: minoring <minho1350@gmail.com>
Date: Sat, 5 Sep 2020 22:11:16 +0900
Subject: [PATCH 149/459] Fix nonlocal checks

Related to #2189
---
 src/symboltable.rs | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index c3662632..67c80183 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -954,10 +954,10 @@ impl SymbolTableBuilder {
         // Some checks:
         let containing = table.symbols.contains_key(name);
         if containing {
+            let symbol = table.symbols.get(name).unwrap();
             // Role already set..
             match role {
                 SymbolUsage::Global => {
-                    let symbol = table.symbols.get(name).unwrap();
                     if let SymbolScope::Global = symbol.scope {
                         // Ok
                     } else {
@@ -968,10 +968,33 @@ impl SymbolTableBuilder {
                     }
                 }
                 SymbolUsage::Nonlocal => {
-                    return Err(SymbolTableError {
-                        error: format!("name '{}' is used prior to nonlocal declaration", name),
-                        location,
-                    })
+                    if symbol.is_parameter {
+                        return Err(SymbolTableError {
+                            error: format!("name '{}' is parameter and nonlocal", name),
+                            location,
+                        });
+                    }
+                    if symbol.is_referenced {
+                        return Err(SymbolTableError {
+                            error: format!("name '{}' is used prior to nonlocal declaration", name),
+                            location,
+                        });
+                    }
+                    if symbol.is_annotated {
+                        return Err(SymbolTableError {
+                            error: format!("annotated name '{}' can't be nonlocal", name),
+                            location,
+                        });
+                    }
+                    if symbol.is_assigned {
+                        return Err(SymbolTableError {
+                            error: format!(
+                                "name '{}' is assigned to before nonlocal declaration",
+                                name
+                            ),
+                            location,
+                        });
+                    }
                 }
                 _ => {
                     // Ok?

From 678b8e779b9152cd20dd8cc7002e9716d3d3bcbe Mon Sep 17 00:00:00 2001
From: minoring <minho1350@gmail.com>
Date: Sat, 5 Sep 2020 22:37:52 +0900
Subject: [PATCH 150/459] Modify `nonlocal` to be declared multiple times

Related to #2183
---
 src/symboltable.rs | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 67c80183..4e7fbef3 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -1025,14 +1025,7 @@ impl SymbolTableBuilder {
         let symbol = table.symbols.get_mut(name).unwrap();
         match role {
             SymbolUsage::Nonlocal => {
-                if let SymbolScope::Unknown = symbol.scope {
-                    symbol.scope = SymbolScope::Nonlocal;
-                } else {
-                    return Err(SymbolTableError {
-                        error: format!("Symbol {} scope cannot be set to nonlocal, since its scope was already determined otherwise.", name),
-                        location,
-                    });
-                }
+                symbol.scope = SymbolScope::Nonlocal;
             }
             SymbolUsage::Imported => {
                 symbol.is_assigned = true;

From 09e4e9a86cf74f800e25c3eb1d0f7afbbffa2cab Mon Sep 17 00:00:00 2001
From: minoring <minho1350@gmail.com>
Date: Sun, 6 Sep 2020 11:40:32 +0900
Subject: [PATCH 151/459] Refactor checks for symbol in register_name

---
 src/symboltable.rs | 36 ++++++++++++++++--------------------
 1 file changed, 16 insertions(+), 20 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 4e7fbef3..7d35f789 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -951,10 +951,8 @@ impl SymbolTableBuilder {
         let table = self.tables.last_mut().unwrap();
         let location = Default::default();
 
-        // Some checks:
-        let containing = table.symbols.contains_key(name);
-        if containing {
-            let symbol = table.symbols.get(name).unwrap();
+        // Some checks for the symbol that present on this scope level:
+        if let Some(symbol) = table.symbols.get(name) {
             // Role already set..
             match role {
                 SymbolUsage::Global => {
@@ -1000,23 +998,21 @@ impl SymbolTableBuilder {
                     // Ok?
                 }
             }
-        }
-
-        // Some more checks:
-        match role {
-            SymbolUsage::Nonlocal if scope_depth < 2 => {
-                return Err(SymbolTableError {
-                    error: format!("cannot define nonlocal '{}' at top level.", name),
-                    location,
-                })
-            }
-            _ => {
-                // Ok!
+        } else {
+            // The symbol does not present on this scope level.
+            // Some checks to insert new symbol into symbol table:
+            match role {
+                SymbolUsage::Nonlocal if scope_depth < 2 => {
+                    return Err(SymbolTableError {
+                        error: format!("cannot define nonlocal '{}' at top level.", name),
+                        location,
+                    })
+                }
+                _ => {
+                    // Ok!
+                }
             }
-        }
-
-        // Insert symbol when required:
-        if !containing {
+            // Insert symbol when required:
             let symbol = Symbol::new(name);
             table.symbols.insert(name.to_owned(), symbol);
         }

From 6c96793f058fe5efea71ec4f9af0831e8f2267a0 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 21 Sep 2020 06:07:20 +0900
Subject: [PATCH 152/459] Fix dis.dis to prints inner constant codes like
 CPython

---
 src/compile.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/compile.rs b/src/compile.rs
index a3b75844..05d07727 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1188,6 +1188,7 @@ impl<O: OutputStream> Compiler<O> {
     }
 
     fn store_docstring(&mut self, doc_str: Option<String>) {
+        // TODO: __doc__ must be default None and no bytecodes unless it is Some
         // Duplicate top of stack (the function or class object)
         self.emit(Instruction::Duplicate);
 

From 09b841416260da65662286f888fdc1ea5ea8bbf3 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Thu, 1 Oct 2020 07:08:26 +0900
Subject: [PATCH 153/459] Fix nightly clippy

---
 src/compile.rs     | 15 ++++-----------
 src/symboltable.rs | 12 ++----------
 2 files changed, 6 insertions(+), 21 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 05d07727..2c04ecc9 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -61,10 +61,7 @@ enum FunctionContext {
 
 impl CompileContext {
     fn in_func(self) -> bool {
-        match self.func {
-            FunctionContext::NoFunction => false,
-            _ => true,
-        }
+        !matches!(self.func, FunctionContext::NoFunction)
     }
 }
 
@@ -2006,13 +2003,9 @@ impl<O: OutputStream> Compiler<O> {
     // a list of expressions on the stack, or a list of tuples.
     fn gather_elements(&mut self, elements: &[ast::Expression]) -> CompileResult<bool> {
         // First determine if we have starred elements:
-        let has_stars = elements.iter().any(|e| {
-            if let ast::ExpressionType::Starred { .. } = &e.node {
-                true
-            } else {
-                false
-            }
-        });
+        let has_stars = elements
+            .iter()
+            .any(|e| matches!(e.node, ast::ExpressionType::Starred { .. }));
 
         for element in elements {
             if let ast::ExpressionType::Starred { value } = &element.node {
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 7d35f789..4d759f9a 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -136,19 +136,11 @@ impl Symbol {
     }
 
     pub fn is_global(&self) -> bool {
-        if let SymbolScope::Global = self.scope {
-            true
-        } else {
-            false
-        }
+        matches!(self.scope, SymbolScope::Global)
     }
 
     pub fn is_local(&self) -> bool {
-        if let SymbolScope::Local = self.scope {
-            true
-        } else {
-            false
-        }
+        matches!(self.scope, SymbolScope::Local)
     }
 }
 

From 16a896d51311ec842b7c48314d675c2d33d9b095 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 28 Sep 2020 23:52:59 -0500
Subject: [PATCH 154/459] Fix accessing __doc__ while inside a class definition

---
 src/compile.rs     | 22 +++++++++++++---------
 src/symboltable.rs |  1 +
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 2c04ecc9..f92eadf3 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -989,7 +989,13 @@ impl<O: OutputStream> Compiler<O> {
 
         // Turn code object into function object:
         self.emit(Instruction::MakeFunction);
-        self.store_docstring(doc_str);
+
+        self.emit(Instruction::Duplicate);
+        self.load_docstring(doc_str);
+        self.emit(Instruction::Rotate { amount: 2 });
+        self.emit(Instruction::StoreAttr {
+            name: "__doc__".to_owned(),
+        });
         self.apply_decorators(decorator_list);
 
         self.store_name(name);
@@ -1109,6 +1115,11 @@ impl<O: OutputStream> Compiler<O> {
             name: "__qualname__".to_owned(),
             scope: bytecode::NameScope::Free,
         });
+        self.load_docstring(doc_str);
+        self.emit(Instruction::StoreName {
+            name: "__doc__".to_owned(),
+            scope: bytecode::NameScope::Free,
+        });
         // setup annotations
         if self.find_ann(body) {
             self.emit(Instruction::SetupAnnotation);
@@ -1175,7 +1186,6 @@ impl<O: OutputStream> Compiler<O> {
             });
         }
 
-        self.store_docstring(doc_str);
         self.apply_decorators(decorator_list);
 
         self.store_name(name);
@@ -1184,10 +1194,9 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
-    fn store_docstring(&mut self, doc_str: Option<String>) {
+    fn load_docstring(&mut self, doc_str: Option<String>) {
         // TODO: __doc__ must be default None and no bytecodes unless it is Some
         // Duplicate top of stack (the function or class object)
-        self.emit(Instruction::Duplicate);
 
         // Doc string value:
         self.emit(Instruction::LoadConst {
@@ -1196,11 +1205,6 @@ impl<O: OutputStream> Compiler<O> {
                 None => bytecode::Constant::None, // set docstring None if not declared
             },
         });
-
-        self.emit(Instruction::Rotate { amount: 2 });
-        self.emit(Instruction::StoreAttr {
-            name: "__doc__".to_owned(),
-        });
     }
 
     fn compile_while(
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 4d759f9a..0ad497c3 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -531,6 +531,7 @@ impl SymbolTableBuilder {
                 self.enter_scope(name, SymbolTableType::Class, statement.location.row());
                 self.register_name("__module__", SymbolUsage::Assigned)?;
                 self.register_name("__qualname__", SymbolUsage::Assigned)?;
+                self.register_name("__doc__", SymbolUsage::Assigned)?;
                 self.scan_statements(body)?;
                 self.leave_scope();
                 self.scan_expressions(bases, &ExpressionContext::Load)?;

From eedbf3aa6cfc63c4e50128648a9f53c26f92f0b2 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 30 Sep 2020 10:47:04 -0500
Subject: [PATCH 155/459] Fix __class__ nonlocal resolution

---
 src/symboltable.rs | 44 ++++++++++++++++++--------------------------
 1 file changed, 18 insertions(+), 26 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 0ad497c3..b9bcb852 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -235,15 +235,7 @@ impl<'a> SymbolTableAnalyzer<'a> {
                     if scope_depth > 0 {
                         // check if the name is already defined in any outer scope
                         // therefore
-                        if scope_depth < 2
-                            || !self
-                                .tables
-                                .iter()
-                                .skip(1) // omit the global scope as it is not non-local
-                                .rev() // revert the order for better performance
-                                .any(|t| t.0.contains_key(&symbol.name))
-                        // true when any of symbol tables contains the name -> then negate
-                        {
+                        if scope_depth < 2 || !self.found_in_outer_scope(symbol) {
                             return Err(SymbolTableError {
                                 error: format!("no binding for nonlocal '{}' found", symbol.name),
                                 location: Default::default(),
@@ -274,27 +266,27 @@ impl<'a> SymbolTableAnalyzer<'a> {
         Ok(())
     }
 
+    fn found_in_outer_scope(&self, symbol: &Symbol) -> bool {
+        // Interesting stuff about the __class__ variable:
+        // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
+        symbol.name == "__class__"
+            || self.tables.iter().skip(1).any(|(symbols, typ)| {
+                *typ != SymbolTableType::Class && symbols.contains_key(&symbol.name)
+            })
+    }
+
     fn analyze_unknown_symbol(&self, symbol: &mut Symbol) {
         if symbol.is_assigned || symbol.is_parameter {
             symbol.scope = SymbolScope::Local;
+        } else if self.found_in_outer_scope(symbol) {
+            // Symbol is in some outer scope.
+            symbol.is_free = true;
+        } else if self.tables.is_empty() {
+            // Don't make assumptions when we don't know.
+            symbol.scope = SymbolScope::Unknown;
         } else {
-            // Interesting stuff about the __class__ variable:
-            // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
-            let found_in_outer_scope = symbol.name == "__class__"
-                || self.tables.iter().skip(1).any(|(symbols, typ)| {
-                    *typ != SymbolTableType::Class && symbols.contains_key(&symbol.name)
-                });
-
-            if found_in_outer_scope {
-                // Symbol is in some outer scope.
-                symbol.is_free = true;
-            } else if self.tables.is_empty() {
-                // Don't make assumptions when we don't know.
-                symbol.scope = SymbolScope::Unknown;
-            } else {
-                // If there are scopes above we can assume global.
-                symbol.scope = SymbolScope::Global;
-            }
+            // If there are scopes above we can assume global.
+            symbol.scope = SymbolScope::Global;
         }
     }
 

From 6d0978226e413af828c114dfdf8e66ff71537671 Mon Sep 17 00:00:00 2001
From: Tom Schuster <evilpies@gmail.com>
Date: Sun, 11 Oct 2020 02:37:25 +0200
Subject: [PATCH 156/459] Basic block support in JIT

---
 src/compile.rs | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index f92eadf3..13eac5e2 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1388,14 +1388,18 @@ impl<O: OutputStream> Compiler<O> {
         self.emit(Instruction::CompareOperation {
             op: to_operator(ops.last().unwrap()),
         });
-        self.emit(Instruction::Jump { target: last_label });
 
-        // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs.
-        self.set_label(break_label);
-        self.emit(Instruction::Rotate { amount: 2 });
-        self.emit(Instruction::Pop);
+        if vals.len() > 2 {
+            self.emit(Instruction::Jump { target: last_label });
+
+            // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs.
+            self.set_label(break_label);
+            self.emit(Instruction::Rotate { amount: 2 });
+            self.emit(Instruction::Pop);
+
+            self.set_label(last_label);
+        }
 
-        self.set_label(last_label);
         Ok(())
     }
 

From ebee651a127a34670918b14a223984e7a6fb457b Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 17 Oct 2020 18:56:30 -0500
Subject: [PATCH 157/459] Update compiler to output codeobj.constants, use
 insta for snapshot testing

---
 Cargo.toml                                    |   3 +
 src/compile.rs                                | 352 +++++-------------
 src/lib.rs                                    |   2 -
 src/output_stream.rs                          |  44 ---
 src/peephole.rs                               | 132 -------
 src/peephole/optimizations.rs                 |  99 -----
 ...compile__tests__constant_optimization.snap | 147 ++++++++
 ...hon_compiler__compile__tests__if_ands.snap |  90 +++++
 ...on_compiler__compile__tests__if_mixed.snap | 109 ++++++
 ...thon_compiler__compile__tests__if_ors.snap |  91 +++++
 10 files changed, 535 insertions(+), 534 deletions(-)
 delete mode 100644 src/output_stream.rs
 delete mode 100644 src/peephole.rs
 delete mode 100644 src/peephole/optimizations.rs
 create mode 100644 src/snapshots/rustpython_compiler__compile__tests__constant_optimization.snap
 create mode 100644 src/snapshots/rustpython_compiler__compile__tests__if_ands.snap
 create mode 100644 src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap
 create mode 100644 src/snapshots/rustpython_compiler__compile__tests__if_ors.snap

diff --git a/Cargo.toml b/Cargo.toml
index 9c73674e..4e184f45 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,3 +15,6 @@ rustpython-parser = { path = "../parser", version = "0.1.1" }
 num-complex = { version = "0.3", features = ["serde"] }
 log = "0.4"
 arrayvec = "0.5"
+
+[dev-dependencies]
+insta = { version = "1.1", features = ["ron"] }
diff --git a/src/compile.rs b/src/compile.rs
index 13eac5e2..d395bc07 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -7,8 +7,6 @@
 
 use crate::error::{CompileError, CompileErrorType};
 pub use crate::mode::Mode;
-use crate::output_stream::{CodeObjectStream, OutputStream};
-use crate::peephole::PeepholeOptimizer;
 use crate::symboltable::{
     make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope, SymbolTable,
 };
@@ -17,13 +15,11 @@ use num_complex::Complex64;
 use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Label};
 use rustpython_parser::{ast, parser};
 
-type BasicOutputStream = PeepholeOptimizer<CodeObjectStream>;
-
 type CompileResult<T> = Result<T, CompileError>;
 
 /// Main structure holding the state of compilation.
-struct Compiler<O: OutputStream = BasicOutputStream> {
-    output_stack: Vec<O>,
+struct Compiler {
+    output_stack: Vec<CodeObject>,
     symbol_table_stack: Vec<SymbolTable>,
     nxt_label: usize,
     source_path: String,
@@ -143,7 +139,7 @@ pub fn compile_program_single(
     })
 }
 
-impl<O: OutputStream> Compiler<O> {
+impl Compiler {
     fn new(opts: CompileOpts, source_path: String) -> Self {
         Compiler {
             output_stack: Vec::new(),
@@ -206,9 +202,7 @@ impl<O: OutputStream> Compiler<O> {
 
         let (statements, doc) = get_doc(&program.statements);
         if let Some(value) = doc {
-            self.emit(Instruction::LoadConst {
-                value: bytecode::Constant::String { value },
-            });
+            self.emit_constant(bytecode::ConstantData::Str { value });
             self.emit(Instruction::StoreName {
                 name: "__doc__".to_owned(),
                 scope: bytecode::NameScope::Global,
@@ -219,9 +213,7 @@ impl<O: OutputStream> Compiler<O> {
         assert_eq!(self.output_stack.len(), size_before);
 
         // Emit None at end:
-        self.emit(Instruction::LoadConst {
-            value: bytecode::Constant::None,
-        });
+        self.emit_constant(bytecode::ConstantData::None);
         self.emit(Instruction::ReturnValue);
         Ok(())
     }
@@ -255,9 +247,7 @@ impl<O: OutputStream> Compiler<O> {
         }
 
         if !emitted_return {
-            self.emit(Instruction::LoadConst {
-                value: bytecode::Constant::None,
-            });
+            self.emit_constant(bytecode::ConstantData::None);
             self.emit(Instruction::ReturnValue);
         }
 
@@ -453,9 +443,7 @@ impl<O: OutputStream> Compiler<O> {
                         if is_async {
                             self.emit(Instruction::BeforeAsyncWith);
                             self.emit(Instruction::GetAwaitable);
-                            self.emit(Instruction::LoadConst {
-                                value: bytecode::Constant::None,
-                            });
+                            self.emit_constant(bytecode::ConstantData::None);
                             self.emit(Instruction::YieldFrom);
                             self.emit(Instruction::SetupAsyncWith { end: end_label });
                         } else {
@@ -486,9 +474,7 @@ impl<O: OutputStream> Compiler<O> {
 
                     if is_async {
                         self.emit(Instruction::GetAwaitable);
-                        self.emit(Instruction::LoadConst {
-                            value: bytecode::Constant::None,
-                        });
+                        self.emit_constant(bytecode::ConstantData::None);
                         self.emit(Instruction::YieldFrom);
                     }
 
@@ -589,7 +575,10 @@ impl<O: OutputStream> Compiler<O> {
                 match value {
                     Some(v) => {
                         if self.ctx.func == FunctionContext::AsyncFunction
-                            && self.current_output().is_generator()
+                            && self
+                                .current_code()
+                                .flags
+                                .contains(bytecode::CodeFlags::IS_GENERATOR)
                         {
                             return Err(self.error_loc(
                                 CompileErrorType::AsyncReturnValue,
@@ -599,9 +588,7 @@ impl<O: OutputStream> Compiler<O> {
                         self.compile_expression(v)?;
                     }
                     None => {
-                        self.emit(Instruction::LoadConst {
-                            value: bytecode::Constant::None,
-                        });
+                        self.emit_constant(bytecode::ConstantData::None);
                     }
                 }
 
@@ -687,10 +674,8 @@ impl<O: OutputStream> Compiler<O> {
         let mut num_kw_only_defaults = 0;
         for (kw, default) in args.kwonlyargs.iter().zip(&args.kw_defaults) {
             if let Some(default) = default {
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::String {
-                        value: kw.arg.clone(),
-                    },
+                self.emit_constant(bytecode::ConstantData::Str {
+                    value: kw.arg.clone(),
                 });
                 self.compile_expression(default)?;
                 num_kw_only_defaults += 1;
@@ -912,9 +897,7 @@ impl<O: OutputStream> Compiler<O> {
                 // the last instruction is a ReturnValue already, we don't need to emit it
             }
             _ => {
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::None,
-                });
+                self.emit_constant(bytecode::ConstantData::None);
                 self.emit(Instruction::ReturnValue);
             }
         }
@@ -928,10 +911,8 @@ impl<O: OutputStream> Compiler<O> {
         // Return annotation:
         if let Some(annotation) = returns {
             // key:
-            self.emit(Instruction::LoadConst {
-                value: bytecode::Constant::String {
-                    value: "return".to_owned(),
-                },
+            self.emit_constant(bytecode::ConstantData::Str {
+                value: "return".to_owned(),
             });
             // value:
             self.compile_expression(annotation)?;
@@ -940,10 +921,8 @@ impl<O: OutputStream> Compiler<O> {
 
         let mut visit_arg_annotation = |arg: &ast::Parameter| -> CompileResult<()> {
             if let Some(annotation) = &arg.annotation {
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::String {
-                        value: arg.arg.to_owned(),
-                    },
+                self.emit_constant(bytecode::ConstantData::Str {
+                    value: arg.arg.to_owned(),
                 });
                 self.compile_expression(&annotation)?;
                 num_annotations += 1;
@@ -976,15 +955,11 @@ impl<O: OutputStream> Compiler<O> {
             code.flags |= bytecode::CodeFlags::IS_COROUTINE;
         }
 
-        self.emit(Instruction::LoadConst {
-            value: bytecode::Constant::Code {
-                code: Box::new(code),
-            },
+        self.emit_constant(bytecode::ConstantData::Code {
+            code: Box::new(code),
         });
-        self.emit(Instruction::LoadConst {
-            value: bytecode::Constant::String {
-                value: qualified_name,
-            },
+        self.emit_constant(bytecode::ConstantData::Str {
+            value: qualified_name,
         });
 
         // Turn code object into function object:
@@ -1106,10 +1081,8 @@ impl<O: OutputStream> Compiler<O> {
             name: "__module__".to_owned(),
             scope: bytecode::NameScope::Free,
         });
-        self.emit(Instruction::LoadConst {
-            value: bytecode::Constant::String {
-                value: qualified_name.clone(),
-            },
+        self.emit_constant(bytecode::ConstantData::Str {
+            value: qualified_name.clone(),
         });
         self.emit(Instruction::StoreName {
             name: "__qualname__".to_owned(),
@@ -1125,33 +1098,25 @@ impl<O: OutputStream> Compiler<O> {
             self.emit(Instruction::SetupAnnotation);
         }
         self.compile_statements(new_body)?;
-        self.emit(Instruction::LoadConst {
-            value: bytecode::Constant::None,
-        });
+        self.emit_constant(bytecode::ConstantData::None);
         self.emit(Instruction::ReturnValue);
 
         let mut code = self.pop_code_object();
-        code.flags &= !bytecode::CodeFlags::NEW_LOCALS;
+        code.flags.remove(bytecode::CodeFlags::NEW_LOCALS);
         self.leave_scope();
 
-        self.emit(Instruction::LoadConst {
-            value: bytecode::Constant::Code {
-                code: Box::new(code),
-            },
+        self.emit_constant(bytecode::ConstantData::Code {
+            code: Box::new(code),
         });
-        self.emit(Instruction::LoadConst {
-            value: bytecode::Constant::String {
-                value: name.to_owned(),
-            },
+        self.emit_constant(bytecode::ConstantData::Str {
+            value: name.to_owned(),
         });
 
         // Turn code object into function object:
         self.emit(Instruction::MakeFunction);
 
-        self.emit(Instruction::LoadConst {
-            value: bytecode::Constant::String {
-                value: qualified_name,
-            },
+        self.emit_constant(bytecode::ConstantData::Str {
+            value: qualified_name,
         });
 
         for base in bases {
@@ -1162,7 +1127,7 @@ impl<O: OutputStream> Compiler<O> {
             let mut kwarg_names = vec![];
             for keyword in keywords {
                 if let Some(name) = &keyword.name {
-                    kwarg_names.push(bytecode::Constant::String {
+                    kwarg_names.push(bytecode::ConstantData::Str {
                         value: name.to_owned(),
                     });
                 } else {
@@ -1172,10 +1137,8 @@ impl<O: OutputStream> Compiler<O> {
                 self.compile_expression(&keyword.value)?;
             }
 
-            self.emit(Instruction::LoadConst {
-                value: bytecode::Constant::Tuple {
-                    elements: kwarg_names,
-                },
+            self.emit_constant(bytecode::ConstantData::Tuple {
+                elements: kwarg_names,
             });
             self.emit(Instruction::CallFunction {
                 typ: CallType::Keyword(2 + keywords.len() + bases.len()),
@@ -1199,11 +1162,9 @@ impl<O: OutputStream> Compiler<O> {
         // Duplicate top of stack (the function or class object)
 
         // Doc string value:
-        self.emit(Instruction::LoadConst {
-            value: match doc_str {
-                Some(doc) => bytecode::Constant::String { value: doc },
-                None => bytecode::Constant::None, // set docstring None if not declared
-            },
+        self.emit_constant(match doc_str {
+            Some(doc) => bytecode::ConstantData::Str { value: doc },
+            None => bytecode::ConstantData::None, // set docstring None if not declared
         });
     }
 
@@ -1273,9 +1234,7 @@ impl<O: OutputStream> Compiler<O> {
                 handler: check_asynciter_label,
             });
             self.emit(Instruction::GetANext);
-            self.emit(Instruction::LoadConst {
-                value: bytecode::Constant::None,
-            });
+            self.emit_constant(bytecode::ConstantData::None);
             self.emit(Instruction::YieldFrom);
             self.compile_store(target)?;
             self.emit(Instruction::PopBlock);
@@ -1428,10 +1387,8 @@ impl<O: OutputStream> Compiler<O> {
                 name: String::from("__annotations__"),
                 scope: bytecode::NameScope::Local,
             });
-            self.emit(Instruction::LoadConst {
-                value: bytecode::Constant::String {
-                    value: name.to_owned(),
-                },
+            self.emit_constant(bytecode::ConstantData::Str {
+                value: name.to_owned(),
             });
             self.emit(Instruction::StoreSubscript);
         } else {
@@ -1730,15 +1687,15 @@ impl<O: OutputStream> Compiler<O> {
             }
             Number { value } => {
                 let const_value = match value {
-                    ast::Number::Integer { value } => bytecode::Constant::Integer {
+                    ast::Number::Integer { value } => bytecode::ConstantData::Integer {
                         value: value.clone(),
                     },
-                    ast::Number::Float { value } => bytecode::Constant::Float { value: *value },
-                    ast::Number::Complex { real, imag } => bytecode::Constant::Complex {
+                    ast::Number::Float { value } => bytecode::ConstantData::Float { value: *value },
+                    ast::Number::Complex { real, imag } => bytecode::ConstantData::Complex {
                         value: Complex64::new(*real, *imag),
                     },
                 };
-                self.emit(Instruction::LoadConst { value: const_value });
+                self.emit_constant(const_value);
             }
             List { elements } => {
                 let size = elements.len();
@@ -1781,9 +1738,7 @@ impl<O: OutputStream> Compiler<O> {
                 self.mark_generator();
                 match value {
                     Some(expression) => self.compile_expression(expression)?,
-                    Option::None => self.emit(Instruction::LoadConst {
-                        value: bytecode::Constant::None,
-                    }),
+                    Option::None => self.emit_constant(bytecode::ConstantData::None),
                 };
                 self.emit(Instruction::YieldValue);
             }
@@ -1793,9 +1748,7 @@ impl<O: OutputStream> Compiler<O> {
                 }
                 self.compile_expression(value)?;
                 self.emit(Instruction::GetAwaitable);
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::None,
-                });
+                self.emit_constant(bytecode::ConstantData::None);
                 self.emit(Instruction::YieldFrom);
             }
             YieldFrom { value } => {
@@ -1811,39 +1764,27 @@ impl<O: OutputStream> Compiler<O> {
                 self.mark_generator();
                 self.compile_expression(value)?;
                 self.emit(Instruction::GetIter);
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::None,
-                });
+                self.emit_constant(bytecode::ConstantData::None);
                 self.emit(Instruction::YieldFrom);
             }
             True => {
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::Boolean { value: true },
-                });
+                self.emit_constant(bytecode::ConstantData::Boolean { value: true });
             }
             False => {
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::Boolean { value: false },
-                });
+                self.emit_constant(bytecode::ConstantData::Boolean { value: false });
             }
             ast::ExpressionType::None => {
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::None,
-                });
+                self.emit_constant(bytecode::ConstantData::None);
             }
             Ellipsis => {
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::Ellipsis,
-                });
+                self.emit_constant(bytecode::ConstantData::Ellipsis);
             }
-            String { value } => {
+            ast::ExpressionType::String { value } => {
                 self.compile_string(value)?;
             }
             Bytes { value } => {
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::Bytes {
-                        value: value.clone(),
-                    },
+                self.emit_constant(bytecode::ConstantData::Bytes {
+                    value: value.clone(),
                 });
             }
             Identifier { name } => {
@@ -1862,14 +1803,10 @@ impl<O: OutputStream> Compiler<O> {
                 self.emit(Instruction::ReturnValue);
                 let code = self.pop_code_object();
                 self.leave_scope();
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::Code {
-                        code: Box::new(code),
-                    },
-                });
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::String { value: name },
+                self.emit_constant(bytecode::ConstantData::Code {
+                    code: Box::new(code),
                 });
+                self.emit_constant(bytecode::ConstantData::Str { value: name });
                 // Turn code object into function object:
                 self.emit(Instruction::MakeFunction);
 
@@ -1916,10 +1853,8 @@ impl<O: OutputStream> Compiler<O> {
                 let mut subsize = 0;
                 for keyword in subkeywords {
                     if let Some(name) = &keyword.name {
-                        self.emit(Instruction::LoadConst {
-                            value: bytecode::Constant::String {
-                                value: name.to_owned(),
-                            },
+                        self.emit_constant(bytecode::ConstantData::Str {
+                            value: name.to_owned(),
                         });
                         self.compile_expression(&keyword.value)?;
                         subsize += 1;
@@ -1980,7 +1915,7 @@ impl<O: OutputStream> Compiler<O> {
                 let mut kwarg_names = vec![];
                 for keyword in keywords {
                     if let Some(name) = &keyword.name {
-                        kwarg_names.push(bytecode::Constant::String {
+                        kwarg_names.push(bytecode::ConstantData::Str {
                             value: name.to_owned(),
                         });
                     } else {
@@ -1990,10 +1925,8 @@ impl<O: OutputStream> Compiler<O> {
                     self.compile_expression(&keyword.value)?;
                 }
 
-                self.emit(Instruction::LoadConst {
-                    value: bytecode::Constant::Tuple {
-                        elements: kwarg_names,
-                    },
+                self.emit_constant(bytecode::ConstantData::Tuple {
+                    elements: kwarg_names,
                 });
                 self.emit(Instruction::CallFunction {
                     typ: CallType::Keyword(count),
@@ -2189,16 +2122,12 @@ impl<O: OutputStream> Compiler<O> {
         self.leave_scope();
 
         // List comprehension code:
-        self.emit(Instruction::LoadConst {
-            value: bytecode::Constant::Code {
-                code: Box::new(code),
-            },
+        self.emit_constant(bytecode::ConstantData::Code {
+            code: Box::new(code),
         });
 
         // List comprehension function name:
-        self.emit(Instruction::LoadConst {
-            value: bytecode::Constant::String { value: name },
-        });
+        self.emit_constant(bytecode::ConstantData::Str { value: name });
 
         // Turn code object into function object:
         self.emit(Instruction::MakeFunction);
@@ -2218,9 +2147,7 @@ impl<O: OutputStream> Compiler<O> {
 
     fn compile_string(&mut self, string: &ast::StringGroup) -> CompileResult<()> {
         if let Some(value) = try_get_constant_string(string) {
-            self.emit(Instruction::LoadConst {
-                value: bytecode::Constant::String { value },
-            });
+            self.emit_constant(bytecode::ConstantData::Str { value });
         } else {
             match string {
                 ast::StringGroup::Joined { values } => {
@@ -2230,10 +2157,8 @@ impl<O: OutputStream> Compiler<O> {
                     self.emit(Instruction::BuildString { size: values.len() })
                 }
                 ast::StringGroup::Constant { value } => {
-                    self.emit(Instruction::LoadConst {
-                        value: bytecode::Constant::String {
-                            value: value.to_owned(),
-                        },
+                    self.emit_constant(bytecode::ConstantData::Str {
+                        value: value.to_owned(),
                     });
                 }
                 ast::StringGroup::FormattedValue {
@@ -2243,10 +2168,8 @@ impl<O: OutputStream> Compiler<O> {
                 } => {
                     match spec {
                         Some(spec) => self.compile_string(spec)?,
-                        None => self.emit(Instruction::LoadConst {
-                            value: bytecode::Constant::String {
-                                value: String::new(),
-                            },
+                        None => self.emit_constant(bytecode::ConstantData::Str {
+                            value: String::new(),
                         }),
                     };
                     self.compile_expression(value)?;
@@ -2312,10 +2235,19 @@ impl<O: OutputStream> Compiler<O> {
     fn emit(&mut self, instruction: Instruction) {
         let location = compile_location(&self.current_source_location);
         // TODO: insert source filename
-        self.current_output().emit(instruction, location);
+        let code = self.current_code();
+        code.instructions.push(instruction);
+        code.locations.push(location);
     }
 
-    fn current_output(&mut self) -> &mut O {
+    fn emit_constant(&mut self, constant: bytecode::ConstantData) {
+        let code = self.current_code();
+        let idx = code.constants.len();
+        code.constants.push(constant);
+        self.emit(Instruction::LoadConst { idx })
+    }
+
+    fn current_code(&mut self) -> &mut CodeObject {
         self.output_stack
             .last_mut()
             .expect("No OutputStream on stack")
@@ -2330,7 +2262,9 @@ impl<O: OutputStream> Compiler<O> {
 
     // Assign current position the given label
     fn set_label(&mut self, label: Label) {
-        self.current_output().set_label(label)
+        let code = self.current_code();
+        let pos = code.instructions.len();
+        code.label_map.insert(label, pos);
     }
 
     fn set_source_location(&mut self, location: ast::Location) {
@@ -2350,7 +2284,7 @@ impl<O: OutputStream> Compiler<O> {
     }
 
     fn mark_generator(&mut self) {
-        self.current_output().mark_generator();
+        self.current_code().flags |= bytecode::CodeFlags::IS_GENERATOR
     }
 }
 
@@ -2404,9 +2338,7 @@ fn compile_conversion_flag(conversion_flag: ast::ConversionFlag) -> bytecode::Co
 mod tests {
     use super::{CompileOpts, Compiler};
     use crate::symboltable::make_symbol_table;
-    use rustpython_bytecode::bytecode::Constant::*;
-    use rustpython_bytecode::bytecode::Instruction::*;
-    use rustpython_bytecode::bytecode::{CodeObject, Label};
+    use rustpython_bytecode::bytecode::CodeObject;
     use rustpython_parser::parser;
 
     fn compile_exec(source: &str) -> CodeObject {
@@ -2421,117 +2353,23 @@ mod tests {
 
     #[test]
     fn test_if_ors() {
-        let code = compile_exec("if True or False or False:\n pass\n");
-        assert_eq!(
-            vec![
-                LoadConst {
-                    value: Boolean { value: true }
-                },
-                JumpIfTrue {
-                    target: Label::new(1)
-                },
-                LoadConst {
-                    value: Boolean { value: false }
-                },
-                JumpIfTrue {
-                    target: Label::new(1)
-                },
-                LoadConst {
-                    value: Boolean { value: false }
-                },
-                JumpIfFalse {
-                    target: Label::new(0)
-                },
-                LoadConst { value: None },
-                ReturnValue
-            ],
-            code.instructions
-        );
+        insta::assert_ron_snapshot!(compile_exec("if True or False or False:\n pass\n"));
     }
 
     #[test]
     fn test_if_ands() {
-        let code = compile_exec("if True and False and False:\n pass\n");
-        assert_eq!(
-            vec![
-                LoadConst {
-                    value: Boolean { value: true }
-                },
-                JumpIfFalse {
-                    target: Label::new(0)
-                },
-                LoadConst {
-                    value: Boolean { value: false }
-                },
-                JumpIfFalse {
-                    target: Label::new(0)
-                },
-                LoadConst {
-                    value: Boolean { value: false }
-                },
-                JumpIfFalse {
-                    target: Label::new(0)
-                },
-                LoadConst { value: None },
-                ReturnValue
-            ],
-            code.instructions
-        );
+        insta::assert_ron_snapshot!(compile_exec("if True and False and False:\n pass\n"));
     }
 
     #[test]
     fn test_if_mixed() {
-        let code = compile_exec("if (True and False) or (False and True):\n pass\n");
-        assert_eq!(
-            vec![
-                LoadConst {
-                    value: Boolean { value: true }
-                },
-                JumpIfFalse {
-                    target: Label::new(2)
-                },
-                LoadConst {
-                    value: Boolean { value: false }
-                },
-                JumpIfTrue {
-                    target: Label::new(1)
-                },
-                LoadConst {
-                    value: Boolean { value: false }
-                },
-                JumpIfFalse {
-                    target: Label::new(0)
-                },
-                LoadConst {
-                    value: Boolean { value: true }
-                },
-                JumpIfFalse {
-                    target: Label::new(0)
-                },
-                LoadConst { value: None },
-                ReturnValue
-            ],
-            code.instructions
-        );
+        insta::assert_ron_snapshot!(compile_exec(
+            "if (True and False) or (False and True):\n pass\n"
+        ));
     }
 
     #[test]
     fn test_constant_optimization() {
-        let code = compile_exec("1 + 2 + 3 + 4\n1.5 * 2.5");
-        assert_eq!(
-            code.instructions,
-            vec![
-                LoadConst {
-                    value: Integer { value: 10.into() }
-                },
-                Pop,
-                LoadConst {
-                    value: Float { value: 3.75 }
-                },
-                Pop,
-                LoadConst { value: None },
-                ReturnValue,
-            ]
-        );
+        insta::assert_ron_snapshot!(compile_exec("1 + 2 + 3 + 4\n1.5 * 2.5"));
     }
 }
diff --git a/src/lib.rs b/src/lib.rs
index 4d469350..415ff613 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -9,6 +9,4 @@ extern crate log;
 pub mod compile;
 pub mod error;
 pub mod mode;
-pub(crate) mod output_stream;
-pub mod peephole;
 pub mod symboltable;
diff --git a/src/output_stream.rs b/src/output_stream.rs
deleted file mode 100644
index 9cc49a94..00000000
--- a/src/output_stream.rs
+++ /dev/null
@@ -1,44 +0,0 @@
-use rustpython_bytecode::bytecode::{CodeFlags, CodeObject, Instruction, Label, Location};
-
-pub trait OutputStream: From<CodeObject> + Into<CodeObject> {
-    /// Output an instruction
-    fn emit(&mut self, instruction: Instruction, location: Location);
-    /// Set a label on an instruction
-    fn set_label(&mut self, label: Label);
-    /// Mark the inner CodeObject as a generator
-    fn mark_generator(&mut self);
-    /// Check to see if the inner CodeObject is a generator
-    fn is_generator(&self) -> bool;
-}
-
-pub struct CodeObjectStream {
-    code: CodeObject,
-}
-
-impl From<CodeObject> for CodeObjectStream {
-    fn from(code: CodeObject) -> Self {
-        CodeObjectStream { code }
-    }
-}
-impl From<CodeObjectStream> for CodeObject {
-    fn from(stream: CodeObjectStream) -> Self {
-        stream.code
-    }
-}
-
-impl OutputStream for CodeObjectStream {
-    fn emit(&mut self, instruction: Instruction, location: Location) {
-        self.code.instructions.push(instruction);
-        self.code.locations.push(location);
-    }
-    fn set_label(&mut self, label: Label) {
-        let position = self.code.instructions.len();
-        self.code.label_map.insert(label, position);
-    }
-    fn mark_generator(&mut self) {
-        self.code.flags |= CodeFlags::IS_GENERATOR;
-    }
-    fn is_generator(&self) -> bool {
-        self.code.flags.contains(CodeFlags::IS_GENERATOR)
-    }
-}
diff --git a/src/peephole.rs b/src/peephole.rs
deleted file mode 100644
index aaec0470..00000000
--- a/src/peephole.rs
+++ /dev/null
@@ -1,132 +0,0 @@
-use crate::output_stream::OutputStream;
-use arrayvec::ArrayVec;
-use rustpython_bytecode::bytecode::{CodeObject, Instruction, Label, Location};
-
-pub mod optimizations;
-
-const PEEPHOLE_BUFFER_SIZE: usize = 20;
-
-pub struct InstructionMetadata {
-    loc: Location,
-    labels: Vec<Label>,
-}
-
-impl From<Vec<InstructionMetadata>> for InstructionMetadata {
-    fn from(metas: Vec<Self>) -> Self {
-        debug_assert!(!metas.is_empty(), "`metas` must not be empty");
-        InstructionMetadata {
-            loc: metas[0].loc,
-            labels: metas
-                .into_iter()
-                .flat_map(|meta| meta.labels.into_iter())
-                .collect(),
-        }
-    }
-}
-impl From<Location> for InstructionMetadata {
-    fn from(loc: Location) -> Self {
-        InstructionMetadata {
-            loc,
-            labels: Vec::new(),
-        }
-    }
-}
-
-pub(crate) struct PeepholeOptimizer<O: OutputStream> {
-    inner: O,
-    buffer: ArrayVec<[(Instruction, InstructionMetadata); PEEPHOLE_BUFFER_SIZE]>,
-}
-
-impl<O: OutputStream> From<CodeObject> for PeepholeOptimizer<O> {
-    fn from(code: CodeObject) -> Self {
-        Self::new(code.into())
-    }
-}
-impl<O: OutputStream> From<PeepholeOptimizer<O>> for CodeObject {
-    fn from(mut peep: PeepholeOptimizer<O>) -> Self {
-        peep.flush();
-        peep.inner.into()
-    }
-}
-
-#[macro_export]
-macro_rules! apply_optimizations {
-    ($buf:expr, $($opt:ident),*$(,)?) => {{
-        $($crate::peephole::optimizations::$opt($buf);)*
-    }};
-}
-
-impl<O: OutputStream> PeepholeOptimizer<O> {
-    pub fn new(inner: O) -> Self {
-        PeepholeOptimizer {
-            inner,
-            buffer: ArrayVec::default(),
-        }
-    }
-
-    fn inner_emit(inner: &mut O, instruction: Instruction, meta: InstructionMetadata) {
-        inner.emit(instruction, meta.loc);
-        for label in meta.labels {
-            inner.set_label(label);
-        }
-    }
-
-    fn push(&mut self, instruction: Instruction, meta: InstructionMetadata) {
-        if self.buffer.is_full() {
-            let (instr, meta) = self.buffer.remove(0);
-            Self::inner_emit(&mut self.inner, instr, meta);
-        }
-        self.buffer.push((instruction, meta));
-    }
-
-    fn pop(&mut self) -> (Instruction, InstructionMetadata) {
-        self.buffer
-            .pop()
-            .expect("Failed to pop instruction from PeepholeOptimizer buffer")
-    }
-
-    fn flush(&mut self) {
-        for (instruction, meta) in self.buffer.drain(..) {
-            Self::inner_emit(&mut self.inner, instruction, meta);
-        }
-    }
-
-    fn optimize(&mut self) {
-        apply_optimizations!(self, operator /* , unpack */);
-    }
-}
-
-impl<O> OutputStream for PeepholeOptimizer<O>
-where
-    O: OutputStream,
-{
-    fn emit(&mut self, instruction: Instruction, loc: Location) {
-        self.push(instruction, loc.into());
-        self.optimize();
-    }
-    fn set_label(&mut self, label: Label) {
-        if let Some(instr) = self.buffer.last_mut() {
-            instr.1.labels.push(label)
-        }
-    }
-    fn mark_generator(&mut self) {
-        self.inner.mark_generator()
-    }
-    fn is_generator(&self) -> bool {
-        self.inner.is_generator()
-    }
-}
-
-impl<O: OutputStream> OptimizationBuffer for PeepholeOptimizer<O> {
-    fn emit(&mut self, instruction: Instruction, meta: InstructionMetadata) {
-        self.push(instruction, meta);
-    }
-    fn pop(&mut self) -> (Instruction, InstructionMetadata) {
-        self.pop()
-    }
-}
-
-pub trait OptimizationBuffer {
-    fn emit(&mut self, instruction: Instruction, meta: InstructionMetadata);
-    fn pop(&mut self) -> (Instruction, InstructionMetadata);
-}
diff --git a/src/peephole/optimizations.rs b/src/peephole/optimizations.rs
deleted file mode 100644
index 461baa7c..00000000
--- a/src/peephole/optimizations.rs
+++ /dev/null
@@ -1,99 +0,0 @@
-use rustpython_bytecode::bytecode::{self, Instruction};
-
-use super::{InstructionMetadata, OptimizationBuffer};
-
-macro_rules! metas {
-    [$($metas:expr),*$(,)?] => {
-        InstructionMetadata::from(vec![$($metas),*])
-    };
-}
-macro_rules! lc {
-    ($name:ident {$($field:tt)*}) => {
-        Instruction::LoadConst {
-            value: bytecode::Constant::$name {$($field)*},
-        }
-    };
-    ($name:ident, $($value:tt)*) => {
-        lc!($name { value: $($value)* })
-    };
-}
-macro_rules! emitconst {
-    ($buf:expr, [$($metas:expr),*$(,)?], $($arg:tt)*) => {
-        $buf.emit(
-            lc!($($arg)*),
-            metas![$($metas),*],
-        )
-    };
-}
-
-pub fn operator(buf: &mut impl OptimizationBuffer) {
-    let (instruction, meta) = buf.pop();
-    if let Instruction::BinaryOperation { op, inplace } = instruction {
-        let (rhs, rhs_meta) = buf.pop();
-        let (lhs, lhs_meta) = buf.pop();
-        macro_rules! op {
-            ($op:ident) => {
-                bytecode::BinaryOperator::$op
-            };
-        }
-        match (op, lhs, rhs) {
-            (op!(Add), lc!(Integer, lhs), lc!(Integer, rhs)) => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Integer, lhs + rhs)
-            }
-            (op!(Subtract), lc!(Integer, lhs), lc!(Integer, rhs)) => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Integer, lhs - rhs)
-            }
-            (op!(Add), lc!(Float, lhs), lc!(Float, rhs)) => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs + rhs)
-            }
-            (op!(Subtract), lc!(Float, lhs), lc!(Float, rhs)) => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs - rhs)
-            }
-            (op!(Multiply), lc!(Float, lhs), lc!(Float, rhs)) => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs * rhs)
-            }
-            (op!(Divide), lc!(Float, lhs), lc!(Float, rhs)) if rhs != 0.0 => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs / rhs)
-            }
-            (op!(Power), lc!(Float, lhs), lc!(Float, rhs)) => {
-                emitconst!(buf, [lhs_meta, rhs_meta], Float, lhs.powf(rhs))
-            }
-            (op!(Add), lc!(String, mut lhs), lc!(String, rhs)) => {
-                lhs.push_str(&rhs);
-                emitconst!(buf, [lhs_meta, rhs_meta], String, lhs);
-            }
-            (op, lhs, rhs) => {
-                buf.emit(lhs, lhs_meta);
-                buf.emit(rhs, rhs_meta);
-                buf.emit(Instruction::BinaryOperation { op, inplace }, meta);
-            }
-        }
-    } else {
-        buf.emit(instruction, meta)
-    }
-}
-
-// TODO: make a version of this that doesn't miscompile `a, b = (1, 2) if True else (3, 4)`
-// pub fn unpack(buf: &mut impl OptimizationBuffer) {
-//     let (instruction, meta) = buf.pop();
-//     if let Instruction::UnpackSequence { size } = instruction {
-//         let (arg, arg_meta) = buf.pop();
-//         match arg {
-//             Instruction::BuildTuple {
-//                 size: tup_size,
-//                 unpack,
-//             } if !unpack && tup_size == size => {
-//                 buf.emit(
-//                     Instruction::Reverse { amount: size },
-//                     vec![arg_meta, meta].into(),
-//                 );
-//             }
-//             arg => {
-//                 buf.emit(arg, arg_meta);
-//                 buf.emit(instruction, meta);
-//             }
-//         }
-//     } else {
-//         buf.emit(instruction, meta)
-//     }
-// }
diff --git a/src/snapshots/rustpython_compiler__compile__tests__constant_optimization.snap b/src/snapshots/rustpython_compiler__compile__tests__constant_optimization.snap
new file mode 100644
index 00000000..abbe9500
--- /dev/null
+++ b/src/snapshots/rustpython_compiler__compile__tests__constant_optimization.snap
@@ -0,0 +1,147 @@
+---
+source: compiler/src/compile.rs
+expression: "compile_exec(\"1 + 2 + 3 + 4\\n1.5 * 2.5\")"
+---
+CodeObject(
+  instructions: [
+    LoadConst(
+      idx: 0,
+    ),
+    LoadConst(
+      idx: 1,
+    ),
+    BinaryOperation(
+      op: Add,
+      inplace: false,
+    ),
+    LoadConst(
+      idx: 2,
+    ),
+    BinaryOperation(
+      op: Add,
+      inplace: false,
+    ),
+    LoadConst(
+      idx: 3,
+    ),
+    BinaryOperation(
+      op: Add,
+      inplace: false,
+    ),
+    Pop,
+    LoadConst(
+      idx: 4,
+    ),
+    LoadConst(
+      idx: 5,
+    ),
+    BinaryOperation(
+      op: Multiply,
+      inplace: false,
+    ),
+    Pop,
+    LoadConst(
+      idx: 6,
+    ),
+    ReturnValue,
+  ],
+  label_map: {},
+  locations: [
+    Location(
+      row: 1,
+      column: 1,
+    ),
+    Location(
+      row: 1,
+      column: 5,
+    ),
+    Location(
+      row: 1,
+      column: 5,
+    ),
+    Location(
+      row: 1,
+      column: 9,
+    ),
+    Location(
+      row: 1,
+      column: 9,
+    ),
+    Location(
+      row: 1,
+      column: 13,
+    ),
+    Location(
+      row: 1,
+      column: 13,
+    ),
+    Location(
+      row: 1,
+      column: 13,
+    ),
+    Location(
+      row: 2,
+      column: 1,
+    ),
+    Location(
+      row: 2,
+      column: 7,
+    ),
+    Location(
+      row: 2,
+      column: 7,
+    ),
+    Location(
+      row: 2,
+      column: 7,
+    ),
+    Location(
+      row: 2,
+      column: 7,
+    ),
+    Location(
+      row: 2,
+      column: 7,
+    ),
+  ],
+  flags: CodeFlags(
+    bits: 8,
+  ),
+  posonlyarg_count: 0,
+  arg_names: [],
+  varargs_name: None,
+  kwonlyarg_names: [],
+  varkeywords_name: None,
+  source_path: "source_path",
+  first_line_number: 0,
+  obj_name: "<module>",
+  constants: [
+    Integer(
+      value: (1, [
+        1,
+      ]),
+    ),
+    Integer(
+      value: (1, [
+        2,
+      ]),
+    ),
+    Integer(
+      value: (1, [
+        3,
+      ]),
+    ),
+    Integer(
+      value: (1, [
+        4,
+      ]),
+    ),
+    Float(
+      value: 1.5,
+    ),
+    Float(
+      value: 2.5,
+    ),
+    None,
+  ],
+)
diff --git a/src/snapshots/rustpython_compiler__compile__tests__if_ands.snap b/src/snapshots/rustpython_compiler__compile__tests__if_ands.snap
new file mode 100644
index 00000000..e89c82e4
--- /dev/null
+++ b/src/snapshots/rustpython_compiler__compile__tests__if_ands.snap
@@ -0,0 +1,90 @@
+---
+source: compiler/src/compile.rs
+expression: "compile_exec(\"if True and False and False:\\n pass\\n\")"
+---
+CodeObject(
+  instructions: [
+    LoadConst(
+      idx: 0,
+    ),
+    JumpIfFalse(
+      target: Label(0),
+    ),
+    LoadConst(
+      idx: 1,
+    ),
+    JumpIfFalse(
+      target: Label(0),
+    ),
+    LoadConst(
+      idx: 2,
+    ),
+    JumpIfFalse(
+      target: Label(0),
+    ),
+    LoadConst(
+      idx: 3,
+    ),
+    ReturnValue,
+  ],
+  label_map: {
+    Label(0): 6,
+  },
+  locations: [
+    Location(
+      row: 1,
+      column: 4,
+    ),
+    Location(
+      row: 1,
+      column: 4,
+    ),
+    Location(
+      row: 1,
+      column: 13,
+    ),
+    Location(
+      row: 1,
+      column: 13,
+    ),
+    Location(
+      row: 1,
+      column: 23,
+    ),
+    Location(
+      row: 1,
+      column: 23,
+    ),
+    Location(
+      row: 2,
+      column: 2,
+    ),
+    Location(
+      row: 2,
+      column: 2,
+    ),
+  ],
+  flags: CodeFlags(
+    bits: 8,
+  ),
+  posonlyarg_count: 0,
+  arg_names: [],
+  varargs_name: None,
+  kwonlyarg_names: [],
+  varkeywords_name: None,
+  source_path: "source_path",
+  first_line_number: 0,
+  obj_name: "<module>",
+  constants: [
+    Boolean(
+      value: true,
+    ),
+    Boolean(
+      value: false,
+    ),
+    Boolean(
+      value: false,
+    ),
+    None,
+  ],
+)
diff --git a/src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap b/src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap
new file mode 100644
index 00000000..302667f1
--- /dev/null
+++ b/src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap
@@ -0,0 +1,109 @@
+---
+source: compiler/src/compile.rs
+expression: "compile_exec(\"if (True and False) or (False and True):\\n pass\\n\")"
+---
+CodeObject(
+  instructions: [
+    LoadConst(
+      idx: 0,
+    ),
+    JumpIfFalse(
+      target: Label(2),
+    ),
+    LoadConst(
+      idx: 1,
+    ),
+    JumpIfTrue(
+      target: Label(1),
+    ),
+    LoadConst(
+      idx: 2,
+    ),
+    JumpIfFalse(
+      target: Label(0),
+    ),
+    LoadConst(
+      idx: 3,
+    ),
+    JumpIfFalse(
+      target: Label(0),
+    ),
+    LoadConst(
+      idx: 4,
+    ),
+    ReturnValue,
+  ],
+  label_map: {
+    Label(0): 8,
+    Label(1): 8,
+    Label(2): 4,
+  },
+  locations: [
+    Location(
+      row: 1,
+      column: 5,
+    ),
+    Location(
+      row: 1,
+      column: 5,
+    ),
+    Location(
+      row: 1,
+      column: 14,
+    ),
+    Location(
+      row: 1,
+      column: 14,
+    ),
+    Location(
+      row: 1,
+      column: 25,
+    ),
+    Location(
+      row: 1,
+      column: 25,
+    ),
+    Location(
+      row: 1,
+      column: 35,
+    ),
+    Location(
+      row: 1,
+      column: 35,
+    ),
+    Location(
+      row: 2,
+      column: 2,
+    ),
+    Location(
+      row: 2,
+      column: 2,
+    ),
+  ],
+  flags: CodeFlags(
+    bits: 8,
+  ),
+  posonlyarg_count: 0,
+  arg_names: [],
+  varargs_name: None,
+  kwonlyarg_names: [],
+  varkeywords_name: None,
+  source_path: "source_path",
+  first_line_number: 0,
+  obj_name: "<module>",
+  constants: [
+    Boolean(
+      value: true,
+    ),
+    Boolean(
+      value: false,
+    ),
+    Boolean(
+      value: false,
+    ),
+    Boolean(
+      value: true,
+    ),
+    None,
+  ],
+)
diff --git a/src/snapshots/rustpython_compiler__compile__tests__if_ors.snap b/src/snapshots/rustpython_compiler__compile__tests__if_ors.snap
new file mode 100644
index 00000000..21a5e47b
--- /dev/null
+++ b/src/snapshots/rustpython_compiler__compile__tests__if_ors.snap
@@ -0,0 +1,91 @@
+---
+source: compiler/src/compile.rs
+expression: "compile_exec(\"if True or False or False:\\n pass\\n\")"
+---
+CodeObject(
+  instructions: [
+    LoadConst(
+      idx: 0,
+    ),
+    JumpIfTrue(
+      target: Label(1),
+    ),
+    LoadConst(
+      idx: 1,
+    ),
+    JumpIfTrue(
+      target: Label(1),
+    ),
+    LoadConst(
+      idx: 2,
+    ),
+    JumpIfFalse(
+      target: Label(0),
+    ),
+    LoadConst(
+      idx: 3,
+    ),
+    ReturnValue,
+  ],
+  label_map: {
+    Label(0): 6,
+    Label(1): 6,
+  },
+  locations: [
+    Location(
+      row: 1,
+      column: 4,
+    ),
+    Location(
+      row: 1,
+      column: 4,
+    ),
+    Location(
+      row: 1,
+      column: 12,
+    ),
+    Location(
+      row: 1,
+      column: 12,
+    ),
+    Location(
+      row: 1,
+      column: 21,
+    ),
+    Location(
+      row: 1,
+      column: 21,
+    ),
+    Location(
+      row: 2,
+      column: 2,
+    ),
+    Location(
+      row: 2,
+      column: 2,
+    ),
+  ],
+  flags: CodeFlags(
+    bits: 8,
+  ),
+  posonlyarg_count: 0,
+  arg_names: [],
+  varargs_name: None,
+  kwonlyarg_names: [],
+  varkeywords_name: None,
+  source_path: "source_path",
+  first_line_number: 0,
+  obj_name: "<module>",
+  constants: [
+    Boolean(
+      value: true,
+    ),
+    Boolean(
+      value: false,
+    ),
+    Boolean(
+      value: false,
+    ),
+    None,
+  ],
+)

From 868fdfcc36159e81640612e37266e286679b2a2d Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 17 Oct 2020 20:05:09 -0500
Subject: [PATCH 158/459] Fix clippy lints

---
 src/compile.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index d395bc07..fde8d05b 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -170,7 +170,7 @@ impl Compiler {
     }
 
     fn push_output(&mut self, code: CodeObject) {
-        self.output_stack.push(code.into());
+        self.output_stack.push(code);
     }
 
     fn push_new_code_object(&mut self, obj_name: String) {
@@ -189,7 +189,7 @@ impl Compiler {
     }
 
     fn pop_code_object(&mut self) -> CodeObject {
-        self.output_stack.pop().unwrap().into()
+        self.output_stack.pop().unwrap()
     }
 
     fn compile_program(

From 3f9dd3dd4c1cf7ef53ad27628bcbc6ae15714d06 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 17 Oct 2020 21:39:05 -0500
Subject: [PATCH 159/459] Remove optimization unit test

---
 src/compile.rs                                |   5 -
 ...compile__tests__constant_optimization.snap | 147 ------------------
 2 files changed, 152 deletions(-)
 delete mode 100644 src/snapshots/rustpython_compiler__compile__tests__constant_optimization.snap

diff --git a/src/compile.rs b/src/compile.rs
index fde8d05b..bbc5123b 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -2367,9 +2367,4 @@ mod tests {
             "if (True and False) or (False and True):\n pass\n"
         ));
     }
-
-    #[test]
-    fn test_constant_optimization() {
-        insta::assert_ron_snapshot!(compile_exec("1 + 2 + 3 + 4\n1.5 * 2.5"));
-    }
 }
diff --git a/src/snapshots/rustpython_compiler__compile__tests__constant_optimization.snap b/src/snapshots/rustpython_compiler__compile__tests__constant_optimization.snap
deleted file mode 100644
index abbe9500..00000000
--- a/src/snapshots/rustpython_compiler__compile__tests__constant_optimization.snap
+++ /dev/null
@@ -1,147 +0,0 @@
----
-source: compiler/src/compile.rs
-expression: "compile_exec(\"1 + 2 + 3 + 4\\n1.5 * 2.5\")"
----
-CodeObject(
-  instructions: [
-    LoadConst(
-      idx: 0,
-    ),
-    LoadConst(
-      idx: 1,
-    ),
-    BinaryOperation(
-      op: Add,
-      inplace: false,
-    ),
-    LoadConst(
-      idx: 2,
-    ),
-    BinaryOperation(
-      op: Add,
-      inplace: false,
-    ),
-    LoadConst(
-      idx: 3,
-    ),
-    BinaryOperation(
-      op: Add,
-      inplace: false,
-    ),
-    Pop,
-    LoadConst(
-      idx: 4,
-    ),
-    LoadConst(
-      idx: 5,
-    ),
-    BinaryOperation(
-      op: Multiply,
-      inplace: false,
-    ),
-    Pop,
-    LoadConst(
-      idx: 6,
-    ),
-    ReturnValue,
-  ],
-  label_map: {},
-  locations: [
-    Location(
-      row: 1,
-      column: 1,
-    ),
-    Location(
-      row: 1,
-      column: 5,
-    ),
-    Location(
-      row: 1,
-      column: 5,
-    ),
-    Location(
-      row: 1,
-      column: 9,
-    ),
-    Location(
-      row: 1,
-      column: 9,
-    ),
-    Location(
-      row: 1,
-      column: 13,
-    ),
-    Location(
-      row: 1,
-      column: 13,
-    ),
-    Location(
-      row: 1,
-      column: 13,
-    ),
-    Location(
-      row: 2,
-      column: 1,
-    ),
-    Location(
-      row: 2,
-      column: 7,
-    ),
-    Location(
-      row: 2,
-      column: 7,
-    ),
-    Location(
-      row: 2,
-      column: 7,
-    ),
-    Location(
-      row: 2,
-      column: 7,
-    ),
-    Location(
-      row: 2,
-      column: 7,
-    ),
-  ],
-  flags: CodeFlags(
-    bits: 8,
-  ),
-  posonlyarg_count: 0,
-  arg_names: [],
-  varargs_name: None,
-  kwonlyarg_names: [],
-  varkeywords_name: None,
-  source_path: "source_path",
-  first_line_number: 0,
-  obj_name: "<module>",
-  constants: [
-    Integer(
-      value: (1, [
-        1,
-      ]),
-    ),
-    Integer(
-      value: (1, [
-        2,
-      ]),
-    ),
-    Integer(
-      value: (1, [
-        3,
-      ]),
-    ),
-    Integer(
-      value: (1, [
-        4,
-      ]),
-    ),
-    Float(
-      value: 1.5,
-    ),
-    Float(
-      value: 2.5,
-    ),
-    None,
-  ],
-)

From 4afbc082ed3e880dce77a996b388bcaa796dc3d4 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 17 Oct 2020 21:54:47 -0500
Subject: [PATCH 160/459] Use disassembly for snapshot testing

---
 Cargo.toml                                    |   2 +-
 src/compile.rs                                |  31 ++++-
 ...hon_compiler__compile__tests__if_ands.snap |  97 ++------------
 ...on_compiler__compile__tests__if_mixed.snap | 118 ++----------------
 ...thon_compiler__compile__tests__if_ors.snap |  98 ++-------------
 5 files changed, 60 insertions(+), 286 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 4e184f45..6ca88818 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -17,4 +17,4 @@ log = "0.4"
 arrayvec = "0.5"
 
 [dev-dependencies]
-insta = { version = "1.1", features = ["ron"] }
+insta = "1.1"
diff --git a/src/compile.rs b/src/compile.rs
index bbc5123b..4be7c307 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -2351,20 +2351,43 @@ mod tests {
         compiler.pop_code_object()
     }
 
+    macro_rules! assert_dis_snapshot {
+        ($value:expr) => {
+            insta::assert_snapshot!(
+                insta::internals::AutoName,
+                $value.display_expand_codeobjects().to_string(),
+                stringify!($value)
+            )
+        };
+    }
+
     #[test]
     fn test_if_ors() {
-        insta::assert_ron_snapshot!(compile_exec("if True or False or False:\n pass\n"));
+        assert_dis_snapshot!(compile_exec(
+            "\
+if True or False or False:
+    pass
+"
+        ));
     }
 
     #[test]
     fn test_if_ands() {
-        insta::assert_ron_snapshot!(compile_exec("if True and False and False:\n pass\n"));
+        assert_dis_snapshot!(compile_exec(
+            "\
+if True and False and False:
+    pass
+"
+        ));
     }
 
     #[test]
     fn test_if_mixed() {
-        insta::assert_ron_snapshot!(compile_exec(
-            "if (True and False) or (False and True):\n pass\n"
+        assert_dis_snapshot!(compile_exec(
+            "\
+if (True and False) or (False and True):
+    pass
+"
         ));
     }
 }
diff --git a/src/snapshots/rustpython_compiler__compile__tests__if_ands.snap b/src/snapshots/rustpython_compiler__compile__tests__if_ands.snap
index e89c82e4..0d5953c2 100644
--- a/src/snapshots/rustpython_compiler__compile__tests__if_ands.snap
+++ b/src/snapshots/rustpython_compiler__compile__tests__if_ands.snap
@@ -1,90 +1,13 @@
 ---
 source: compiler/src/compile.rs
-expression: "compile_exec(\"if True and False and False:\\n pass\\n\")"
+expression: "compile_exec(\"\\\nif True and False and False:\n    pass\n\")"
 ---
-CodeObject(
-  instructions: [
-    LoadConst(
-      idx: 0,
-    ),
-    JumpIfFalse(
-      target: Label(0),
-    ),
-    LoadConst(
-      idx: 1,
-    ),
-    JumpIfFalse(
-      target: Label(0),
-    ),
-    LoadConst(
-      idx: 2,
-    ),
-    JumpIfFalse(
-      target: Label(0),
-    ),
-    LoadConst(
-      idx: 3,
-    ),
-    ReturnValue,
-  ],
-  label_map: {
-    Label(0): 6,
-  },
-  locations: [
-    Location(
-      row: 1,
-      column: 4,
-    ),
-    Location(
-      row: 1,
-      column: 4,
-    ),
-    Location(
-      row: 1,
-      column: 13,
-    ),
-    Location(
-      row: 1,
-      column: 13,
-    ),
-    Location(
-      row: 1,
-      column: 23,
-    ),
-    Location(
-      row: 1,
-      column: 23,
-    ),
-    Location(
-      row: 2,
-      column: 2,
-    ),
-    Location(
-      row: 2,
-      column: 2,
-    ),
-  ],
-  flags: CodeFlags(
-    bits: 8,
-  ),
-  posonlyarg_count: 0,
-  arg_names: [],
-  varargs_name: None,
-  kwonlyarg_names: [],
-  varkeywords_name: None,
-  source_path: "source_path",
-  first_line_number: 0,
-  obj_name: "<module>",
-  constants: [
-    Boolean(
-      value: true,
-    ),
-    Boolean(
-      value: false,
-    ),
-    Boolean(
-      value: false,
-    ),
-    None,
-  ],
-)
+                 0 LoadConst            (true)
+                 1 JumpIfFalse          (6)
+                 2 LoadConst            (false)
+                 3 JumpIfFalse          (6)
+                 4 LoadConst            (false)
+                 5 JumpIfFalse          (6)
+          >>     6 LoadConst            (None)
+                 7 ReturnValue
+
diff --git a/src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap b/src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap
index 302667f1..5476053b 100644
--- a/src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap
+++ b/src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap
@@ -1,109 +1,15 @@
 ---
 source: compiler/src/compile.rs
-expression: "compile_exec(\"if (True and False) or (False and True):\\n pass\\n\")"
+expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n    pass\n\")"
 ---
-CodeObject(
-  instructions: [
-    LoadConst(
-      idx: 0,
-    ),
-    JumpIfFalse(
-      target: Label(2),
-    ),
-    LoadConst(
-      idx: 1,
-    ),
-    JumpIfTrue(
-      target: Label(1),
-    ),
-    LoadConst(
-      idx: 2,
-    ),
-    JumpIfFalse(
-      target: Label(0),
-    ),
-    LoadConst(
-      idx: 3,
-    ),
-    JumpIfFalse(
-      target: Label(0),
-    ),
-    LoadConst(
-      idx: 4,
-    ),
-    ReturnValue,
-  ],
-  label_map: {
-    Label(0): 8,
-    Label(1): 8,
-    Label(2): 4,
-  },
-  locations: [
-    Location(
-      row: 1,
-      column: 5,
-    ),
-    Location(
-      row: 1,
-      column: 5,
-    ),
-    Location(
-      row: 1,
-      column: 14,
-    ),
-    Location(
-      row: 1,
-      column: 14,
-    ),
-    Location(
-      row: 1,
-      column: 25,
-    ),
-    Location(
-      row: 1,
-      column: 25,
-    ),
-    Location(
-      row: 1,
-      column: 35,
-    ),
-    Location(
-      row: 1,
-      column: 35,
-    ),
-    Location(
-      row: 2,
-      column: 2,
-    ),
-    Location(
-      row: 2,
-      column: 2,
-    ),
-  ],
-  flags: CodeFlags(
-    bits: 8,
-  ),
-  posonlyarg_count: 0,
-  arg_names: [],
-  varargs_name: None,
-  kwonlyarg_names: [],
-  varkeywords_name: None,
-  source_path: "source_path",
-  first_line_number: 0,
-  obj_name: "<module>",
-  constants: [
-    Boolean(
-      value: true,
-    ),
-    Boolean(
-      value: false,
-    ),
-    Boolean(
-      value: false,
-    ),
-    Boolean(
-      value: true,
-    ),
-    None,
-  ],
-)
+                 0 LoadConst            (true)
+                 1 JumpIfFalse          (4)
+                 2 LoadConst            (false)
+                 3 JumpIfTrue           (8)
+          >>     4 LoadConst            (false)
+                 5 JumpIfFalse          (8)
+                 6 LoadConst            (true)
+                 7 JumpIfFalse          (8)
+          >>     8 LoadConst            (None)
+                 9 ReturnValue
+
diff --git a/src/snapshots/rustpython_compiler__compile__tests__if_ors.snap b/src/snapshots/rustpython_compiler__compile__tests__if_ors.snap
index 21a5e47b..d2bc3850 100644
--- a/src/snapshots/rustpython_compiler__compile__tests__if_ors.snap
+++ b/src/snapshots/rustpython_compiler__compile__tests__if_ors.snap
@@ -1,91 +1,13 @@
 ---
 source: compiler/src/compile.rs
-expression: "compile_exec(\"if True or False or False:\\n pass\\n\")"
+expression: "compile_exec(\"\\\nif True or False or False:\n    pass\n\")"
 ---
-CodeObject(
-  instructions: [
-    LoadConst(
-      idx: 0,
-    ),
-    JumpIfTrue(
-      target: Label(1),
-    ),
-    LoadConst(
-      idx: 1,
-    ),
-    JumpIfTrue(
-      target: Label(1),
-    ),
-    LoadConst(
-      idx: 2,
-    ),
-    JumpIfFalse(
-      target: Label(0),
-    ),
-    LoadConst(
-      idx: 3,
-    ),
-    ReturnValue,
-  ],
-  label_map: {
-    Label(0): 6,
-    Label(1): 6,
-  },
-  locations: [
-    Location(
-      row: 1,
-      column: 4,
-    ),
-    Location(
-      row: 1,
-      column: 4,
-    ),
-    Location(
-      row: 1,
-      column: 12,
-    ),
-    Location(
-      row: 1,
-      column: 12,
-    ),
-    Location(
-      row: 1,
-      column: 21,
-    ),
-    Location(
-      row: 1,
-      column: 21,
-    ),
-    Location(
-      row: 2,
-      column: 2,
-    ),
-    Location(
-      row: 2,
-      column: 2,
-    ),
-  ],
-  flags: CodeFlags(
-    bits: 8,
-  ),
-  posonlyarg_count: 0,
-  arg_names: [],
-  varargs_name: None,
-  kwonlyarg_names: [],
-  varkeywords_name: None,
-  source_path: "source_path",
-  first_line_number: 0,
-  obj_name: "<module>",
-  constants: [
-    Boolean(
-      value: true,
-    ),
-    Boolean(
-      value: false,
-    ),
-    Boolean(
-      value: false,
-    ),
-    None,
-  ],
-)
+                 0 LoadConst            (true)
+                 1 JumpIfTrue           (6)
+                 2 LoadConst            (false)
+                 3 JumpIfTrue           (6)
+                 4 LoadConst            (false)
+                 5 JumpIfFalse          (6)
+          >>     6 LoadConst            (None)
+                 7 ReturnValue
+

From 3f88b08aaaf768b11be0e34303f088551ef18baf Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 30 Oct 2020 14:05:53 -0500
Subject: [PATCH 161/459] Capitalize boolean constants for dis

---
 .../rustpython_compiler__compile__tests__if_ands.snap     | 6 +++---
 .../rustpython_compiler__compile__tests__if_mixed.snap    | 8 ++++----
 .../rustpython_compiler__compile__tests__if_ors.snap      | 6 +++---
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/snapshots/rustpython_compiler__compile__tests__if_ands.snap b/src/snapshots/rustpython_compiler__compile__tests__if_ands.snap
index 0d5953c2..d3344e3d 100644
--- a/src/snapshots/rustpython_compiler__compile__tests__if_ands.snap
+++ b/src/snapshots/rustpython_compiler__compile__tests__if_ands.snap
@@ -2,11 +2,11 @@
 source: compiler/src/compile.rs
 expression: "compile_exec(\"\\\nif True and False and False:\n    pass\n\")"
 ---
-                 0 LoadConst            (true)
+                 0 LoadConst            (True)
                  1 JumpIfFalse          (6)
-                 2 LoadConst            (false)
+                 2 LoadConst            (False)
                  3 JumpIfFalse          (6)
-                 4 LoadConst            (false)
+                 4 LoadConst            (False)
                  5 JumpIfFalse          (6)
           >>     6 LoadConst            (None)
                  7 ReturnValue
diff --git a/src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap b/src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap
index 5476053b..356067af 100644
--- a/src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap
+++ b/src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap
@@ -2,13 +2,13 @@
 source: compiler/src/compile.rs
 expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n    pass\n\")"
 ---
-                 0 LoadConst            (true)
+                 0 LoadConst            (True)
                  1 JumpIfFalse          (4)
-                 2 LoadConst            (false)
+                 2 LoadConst            (False)
                  3 JumpIfTrue           (8)
-          >>     4 LoadConst            (false)
+          >>     4 LoadConst            (False)
                  5 JumpIfFalse          (8)
-                 6 LoadConst            (true)
+                 6 LoadConst            (True)
                  7 JumpIfFalse          (8)
           >>     8 LoadConst            (None)
                  9 ReturnValue
diff --git a/src/snapshots/rustpython_compiler__compile__tests__if_ors.snap b/src/snapshots/rustpython_compiler__compile__tests__if_ors.snap
index d2bc3850..4463c3bd 100644
--- a/src/snapshots/rustpython_compiler__compile__tests__if_ors.snap
+++ b/src/snapshots/rustpython_compiler__compile__tests__if_ors.snap
@@ -2,11 +2,11 @@
 source: compiler/src/compile.rs
 expression: "compile_exec(\"\\\nif True or False or False:\n    pass\n\")"
 ---
-                 0 LoadConst            (true)
+                 0 LoadConst            (True)
                  1 JumpIfTrue           (6)
-                 2 LoadConst            (false)
+                 2 LoadConst            (False)
                  3 JumpIfTrue           (6)
-                 4 LoadConst            (false)
+                 4 LoadConst            (False)
                  5 JumpIfFalse          (6)
           >>     6 LoadConst            (None)
                  7 ReturnValue

From e9095a741d4c0355d77add60dbc0054de2bf9b78 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 2 Nov 2020 12:37:19 -0600
Subject: [PATCH 162/459] Split the ast from the parser, remove compiler dep on
 parser

---
 Cargo.toml         |   3 +-
 src/compile.rs     |  46 +++-------
 src/error.rs       | 110 +++++++-----------------
 src/mode.rs        |  11 ---
 src/symboltable.rs | 206 +++++++++++++++++++++++----------------------
 5 files changed, 152 insertions(+), 224 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 6ca88818..dd9f1d0b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,10 +11,11 @@ edition = "2018"
 indexmap = "1.0"
 itertools = "0.9"
 rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
-rustpython-parser = { path = "../parser", version = "0.1.1" }
+rustpython-ast = { path = "../ast" }
 num-complex = { version = "0.3", features = ["serde"] }
 log = "0.4"
 arrayvec = "0.5"
 
 [dev-dependencies]
+rustpython-parser = { path = "../parser" }
 insta = "1.1"
diff --git a/src/compile.rs b/src/compile.rs
index 4be7c307..6c371af5 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -12,8 +12,8 @@ use crate::symboltable::{
 };
 use itertools::Itertools;
 use num_complex::Complex64;
+use rustpython_ast as ast;
 use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Label};
-use rustpython_parser::{ast, parser};
 
 type CompileResult<T> = Result<T, CompileError>;
 
@@ -61,31 +61,6 @@ impl CompileContext {
     }
 }
 
-/// Compile a given sourcecode into a bytecode object.
-pub fn compile(
-    source: &str,
-    mode: Mode,
-    source_path: String,
-    opts: CompileOpts,
-) -> CompileResult<CodeObject> {
-    let to_compile_error =
-        |parse_error| CompileError::from_parse_error(parse_error, source_path.clone());
-    match mode {
-        Mode::Exec => {
-            let ast = parser::parse_program(source).map_err(to_compile_error)?;
-            compile_program(ast, source_path, opts)
-        }
-        Mode::Eval => {
-            let statement = parser::parse_statement(source).map_err(to_compile_error)?;
-            compile_statement_eval(statement, source_path, opts)
-        }
-        Mode::Single => {
-            let ast = parser::parse_program(source).map_err(to_compile_error)?;
-            compile_program_single(ast, source_path, opts)
-        }
-    }
-}
-
 /// A helper function for the shared code of the different compile functions
 fn with_compiler(
     source_path: String,
@@ -106,8 +81,10 @@ pub fn compile_program(
     source_path: String,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    let symbol_table = make_symbol_table(&ast)
-        .map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?;
+    let symbol_table = match make_symbol_table(&ast) {
+        Ok(x) => x,
+        Err(e) => return Err(e.into_compile_error(source_path)),
+    };
     with_compiler(source_path, opts, |compiler| {
         compiler.compile_program(&ast, symbol_table)
     })
@@ -119,8 +96,10 @@ pub fn compile_statement_eval(
     source_path: String,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    let symbol_table = statements_to_symbol_table(&statement)
-        .map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?;
+    let symbol_table = match statements_to_symbol_table(&statement) {
+        Ok(x) => x,
+        Err(e) => return Err(e.into_compile_error(source_path)),
+    };
     with_compiler(source_path, opts, |compiler| {
         compiler.compile_statement_eval(&statement, symbol_table)
     })
@@ -132,8 +111,10 @@ pub fn compile_program_single(
     source_path: String,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    let symbol_table = make_symbol_table(&ast)
-        .map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?;
+    let symbol_table = match make_symbol_table(&ast) {
+        Ok(x) => x,
+        Err(e) => return Err(e.into_compile_error(source_path)),
+    };
     with_compiler(source_path, opts, |compiler| {
         compiler.compile_program_single(&ast, symbol_table)
     })
@@ -165,7 +146,6 @@ impl Compiler {
             error,
             location,
             source_path: self.source_path.clone(),
-            statement: None,
         }
     }
 
diff --git a/src/error.rs b/src/error.rs
index 35a7828f..37881f5c 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -1,34 +1,17 @@
-use rustpython_parser::error::{LexicalErrorType, ParseError, ParseErrorType};
-use rustpython_parser::location::Location;
-use rustpython_parser::token::Tok;
+use rustpython_ast::Location;
 
 use std::error::Error;
 use std::fmt;
 
 #[derive(Debug)]
 pub struct CompileError {
-    pub statement: Option<String>,
     pub error: CompileErrorType,
     pub location: Location,
     pub source_path: String,
 }
 
-impl CompileError {
-    pub fn from_parse_error(parse_error: ParseError, source_path: String) -> Self {
-        Self {
-            statement: None,
-            error: CompileErrorType::Parse(parse_error.error),
-            location: parse_error.location,
-            source_path,
-        }
-    }
-
-    pub fn update_statement_info(&mut self, statement: String) {
-        self.statement = Some(statement);
-    }
-}
-
 #[derive(Debug)]
+#[non_exhaustive]
 pub enum CompileErrorType {
     /// Invalid assignment, cannot store value in target.
     Assign(&'static str),
@@ -36,8 +19,6 @@ pub enum CompileErrorType {
     Delete(&'static str),
     /// Expected an expression got a statement
     ExpectExpr,
-    /// Parser error
-    Parse(ParseErrorType),
     SyntaxError(String),
     /// Multiple `*` detected
     MultipleStarArgs,
@@ -57,74 +38,43 @@ pub enum CompileErrorType {
     InvalidFutureFeature(String),
 }
 
-impl CompileError {
-    pub fn is_indentation_error(&self) -> bool {
-        if let CompileErrorType::Parse(parse) = &self.error {
-            match parse {
-                ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true,
-                ParseErrorType::UnrecognizedToken(token, expected) => {
-                    *token == Tok::Indent || expected.clone() == Some("Indent".to_owned())
-                }
-                _ => false,
-            }
-        } else {
-            false
-        }
-    }
-
-    pub fn is_tab_error(&self) -> bool {
-        if let CompileErrorType::Parse(parse) = &self.error {
-            if let ParseErrorType::Lexical(lex) = parse {
-                if let LexicalErrorType::TabError = lex {
-                    return true;
-                }
-            }
-        }
-        false
-    }
-}
-
-impl fmt::Display for CompileError {
+impl fmt::Display for CompileErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let error_desc = match &self.error {
-            CompileErrorType::Assign(target) => format!("can't assign to {}", target),
-            CompileErrorType::Delete(target) => format!("can't delete {}", target),
-            CompileErrorType::ExpectExpr => "Expecting expression, got statement".to_owned(),
-            CompileErrorType::Parse(err) => err.to_string(),
-            CompileErrorType::SyntaxError(err) => err.to_string(),
+        match self {
+            CompileErrorType::Assign(target) => write!(f, "can't assign to {}", target),
+            CompileErrorType::Delete(target) => write!(f, "can't delete {}", target),
+            CompileErrorType::ExpectExpr => write!(f, "Expecting expression, got statement"),
+            CompileErrorType::SyntaxError(err) => write!(f, "{}", err.as_str()),
             CompileErrorType::MultipleStarArgs => {
-                "two starred expressions in assignment".to_owned()
+                write!(f, "two starred expressions in assignment")
             }
-            CompileErrorType::InvalidStarExpr => "can't use starred expression here".to_owned(),
-            CompileErrorType::InvalidBreak => "'break' outside loop".to_owned(),
-            CompileErrorType::InvalidContinue => "'continue' outside loop".to_owned(),
-            CompileErrorType::InvalidReturn => "'return' outside function".to_owned(),
-            CompileErrorType::InvalidYield => "'yield' outside function".to_owned(),
-            CompileErrorType::InvalidYieldFrom => "'yield from' outside function".to_owned(),
-            CompileErrorType::InvalidAwait => "'await' outside async function".to_owned(),
-            CompileErrorType::AsyncYieldFrom => "'yield from' inside async function".to_owned(),
+            CompileErrorType::InvalidStarExpr => write!(f, "can't use starred expression here"),
+            CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"),
+            CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"),
+            CompileErrorType::InvalidReturn => write!(f, "'return' outside function"),
+            CompileErrorType::InvalidYield => write!(f, "'yield' outside function"),
+            CompileErrorType::InvalidYieldFrom => write!(f, "'yield from' outside function"),
+            CompileErrorType::InvalidAwait => write!(f, "'await' outside async function"),
+            CompileErrorType::AsyncYieldFrom => write!(f, "'yield from' inside async function"),
             CompileErrorType::AsyncReturnValue => {
-                "'return' with value inside async generator".to_owned()
-            }
-            CompileErrorType::InvalidFuturePlacement => {
-                "from __future__ imports must occur at the beginning of the file".to_owned()
+                write!(f, "'return' with value inside async generator")
             }
+            CompileErrorType::InvalidFuturePlacement => write!(
+                f,
+                "from __future__ imports must occur at the beginning of the file"
+            ),
             CompileErrorType::InvalidFutureFeature(feat) => {
-                format!("future feature {} is not defined", feat)
-            }
-        };
-
-        if let Some(statement) = &self.statement {
-            if self.location.column() > 0 {
-                if let Some(line) = statement.lines().nth(self.location.row() - 1) {
-                    // visualize the error, when location and statement are provided
-                    return write!(f, "{}", self.location.visualize(line, &error_desc));
-                }
+                write!(f, "future feature {} is not defined", feat)
             }
         }
+    }
+}
 
-        // print line number
-        write!(f, "{} at {}", error_desc, self.location)
+impl Error for CompileErrorType {}
+
+impl fmt::Display for CompileError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{} at {}", self.error, self.location)
     }
 }
 
diff --git a/src/mode.rs b/src/mode.rs
index e6ca51f3..f926309a 100644
--- a/src/mode.rs
+++ b/src/mode.rs
@@ -1,5 +1,3 @@
-use rustpython_parser::parser;
-
 #[derive(Clone, Copy)]
 pub enum Mode {
     Exec,
@@ -19,15 +17,6 @@ impl std::str::FromStr for Mode {
     }
 }
 
-impl Mode {
-    pub fn to_parser_mode(self) -> parser::Mode {
-        match self {
-            Mode::Exec | Mode::Single => parser::Mode::Program,
-            Mode::Eval => parser::Mode::Statement,
-        }
-    }
-}
-
 #[derive(Debug)]
 pub struct ModeParseError {
     _priv: (),
diff --git a/src/symboltable.rs b/src/symboltable.rs
index b9bcb852..276e55c4 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -9,12 +9,11 @@ Inspirational file: https://github.com/python/cpython/blob/master/Python/symtabl
 
 use crate::error::{CompileError, CompileErrorType};
 use indexmap::map::IndexMap;
-use rustpython_parser::ast;
-use rustpython_parser::location::Location;
+use rustpython_ast::{self as ast, Location};
 use std::fmt;
 
 pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolTable, SymbolTableError> {
-    let mut builder: SymbolTableBuilder = Default::default();
+    let mut builder = SymbolTableBuilder::default();
     builder.prepare();
     builder.scan_program(program)?;
     builder.finish()
@@ -23,7 +22,7 @@ pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolTable, SymbolTa
 pub fn statements_to_symbol_table(
     statements: &[ast::Statement],
 ) -> Result<SymbolTable, SymbolTableError> {
-    let mut builder: SymbolTableBuilder = Default::default();
+    let mut builder = SymbolTableBuilder::default();
     builder.prepare();
     builder.scan_statements(statements)?;
     builder.finish()
@@ -59,7 +58,7 @@ impl SymbolTable {
             typ,
             line_number,
             is_nested,
-            symbols: Default::default(),
+            symbols: IndexMap::new(),
             sub_tables: vec![],
         }
     }
@@ -150,12 +149,11 @@ pub struct SymbolTableError {
     location: Location,
 }
 
-impl CompileError {
-    pub fn from_symbol_table_error(error: SymbolTableError, source_path: String) -> Self {
+impl SymbolTableError {
+    pub fn into_compile_error(self, source_path: String) -> CompileError {
         CompileError {
-            statement: None,
-            error: CompileErrorType::SyntaxError(error.error),
-            location: error.location,
+            error: CompileErrorType::SyntaxError(self.error),
+            location: self.location,
             source_path,
         }
     }
@@ -238,7 +236,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
                         if scope_depth < 2 || !self.found_in_outer_scope(symbol) {
                             return Err(SymbolTableError {
                                 error: format!("no binding for nonlocal '{}' found", symbol.name),
-                                location: Default::default(),
+                                // TODO: accurate location info, somehow
+                                location: Location::default(),
                             });
                         }
                     } else {
@@ -247,7 +246,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
                                 "nonlocal {} defined at place without an enclosing scope",
                                 symbol.name
                             ),
-                            location: Default::default(),
+                            // TODO: accurate location info, somehow
+                            location: Location::default(),
                         });
                     }
                 }
@@ -312,7 +312,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
                     "assignment expression cannot rebind comprehension iteration variable {}",
                     symbol.name
                 ),
-                location: Default::default(),
+                // TODO: accurate location info, somehow
+                location: Location::default(),
             });
         }
 
@@ -324,7 +325,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
                 // named expressions are forbidden in comprehensions on class scope
                 return Err(SymbolTableError {
                     error: "assignment expression within a comprehension cannot be used in a class body".to_string(),
-                    location: Default::default(),
+                    // TODO: accurate location info, somehow
+                    location: Location::default(),
                 } );
             }
             SymbolTableType::Function => {
@@ -356,7 +358,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
                         if parent_symbol.is_iter {
                             return Err(SymbolTableError {
                                 error: format!("assignment expression cannot rebind comprehension iteration variable {}", symbol.name),
-                                location: Default::default(),
+                                // TODO: accurate location info, somehow
+                                location: Location::default(),
                             });
                         }
 
@@ -405,6 +408,7 @@ struct SymbolTableBuilder {
 /// was used.
 /// In cpython this is stored in the AST, but I think this
 /// is not logical, since it is not context free.
+#[derive(Copy, Clone)]
 enum ExpressionContext {
     Load,
     Store,
@@ -466,7 +470,7 @@ impl SymbolTableBuilder {
         } else {
             SymbolUsage::Parameter
         };
-        self.register_name(&parameter.arg, usage)
+        self.register_name(&parameter.arg, usage, parameter.location)
     }
 
     fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
@@ -478,22 +482,23 @@ impl SymbolTableBuilder {
 
     fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
         if let Some(annotation) = &parameter.annotation {
-            self.scan_expression(&annotation, &ExpressionContext::Load)?;
+            self.scan_expression(&annotation, ExpressionContext::Load)?;
         }
         Ok(())
     }
 
     fn scan_statement(&mut self, statement: &ast::Statement) -> SymbolTableResult {
         use ast::StatementType::*;
+        let location = statement.location;
         match &statement.node {
             Global { names } => {
                 for name in names {
-                    self.register_name(name, SymbolUsage::Global)?;
+                    self.register_name(name, SymbolUsage::Global, location)?;
                 }
             }
             Nonlocal { names } => {
                 for name in names {
-                    self.register_name(name, SymbolUsage::Nonlocal)?;
+                    self.register_name(name, SymbolUsage::Nonlocal, location)?;
                 }
             }
             FunctionDef {
@@ -504,12 +509,12 @@ impl SymbolTableBuilder {
                 returns,
                 ..
             } => {
-                self.scan_expressions(decorator_list, &ExpressionContext::Load)?;
-                self.register_name(name, SymbolUsage::Assigned)?;
+                self.scan_expressions(decorator_list, ExpressionContext::Load)?;
+                self.register_name(name, SymbolUsage::Assigned, location)?;
                 if let Some(expression) = returns {
-                    self.scan_expression(expression, &ExpressionContext::Load)?;
+                    self.scan_expression(expression, ExpressionContext::Load)?;
                 }
-                self.enter_function(name, args, statement.location.row())?;
+                self.enter_function(name, args, location.row())?;
                 self.scan_statements(body)?;
                 self.leave_scope();
             }
@@ -520,24 +525,24 @@ impl SymbolTableBuilder {
                 keywords,
                 decorator_list,
             } => {
-                self.enter_scope(name, SymbolTableType::Class, statement.location.row());
-                self.register_name("__module__", SymbolUsage::Assigned)?;
-                self.register_name("__qualname__", SymbolUsage::Assigned)?;
-                self.register_name("__doc__", SymbolUsage::Assigned)?;
+                self.enter_scope(name, SymbolTableType::Class, location.row());
+                self.register_name("__module__", SymbolUsage::Assigned, location)?;
+                self.register_name("__qualname__", SymbolUsage::Assigned, location)?;
+                self.register_name("__doc__", SymbolUsage::Assigned, location)?;
                 self.scan_statements(body)?;
                 self.leave_scope();
-                self.scan_expressions(bases, &ExpressionContext::Load)?;
+                self.scan_expressions(bases, ExpressionContext::Load)?;
                 for keyword in keywords {
-                    self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
+                    self.scan_expression(&keyword.value, ExpressionContext::Load)?;
                 }
-                self.scan_expressions(decorator_list, &ExpressionContext::Load)?;
-                self.register_name(name, SymbolUsage::Assigned)?;
+                self.scan_expressions(decorator_list, ExpressionContext::Load)?;
+                self.register_name(name, SymbolUsage::Assigned, location)?;
             }
             Expression { expression } => {
-                self.scan_expression(expression, &ExpressionContext::Load)?
+                self.scan_expression(expression, ExpressionContext::Load)?
             }
             If { test, body, orelse } => {
-                self.scan_expression(test, &ExpressionContext::Load)?;
+                self.scan_expression(test, ExpressionContext::Load)?;
                 self.scan_statements(body)?;
                 if let Some(code) = orelse {
                     self.scan_statements(code)?;
@@ -550,15 +555,15 @@ impl SymbolTableBuilder {
                 orelse,
                 ..
             } => {
-                self.scan_expression(target, &ExpressionContext::Store)?;
-                self.scan_expression(iter, &ExpressionContext::Load)?;
+                self.scan_expression(target, ExpressionContext::Store)?;
+                self.scan_expression(iter, ExpressionContext::Load)?;
                 self.scan_statements(body)?;
                 if let Some(code) = orelse {
                     self.scan_statements(code)?;
                 }
             }
             While { test, body, orelse } => {
-                self.scan_expression(test, &ExpressionContext::Load)?;
+                self.scan_expression(test, ExpressionContext::Load)?;
                 self.scan_statements(body)?;
                 if let Some(code) = orelse {
                     self.scan_statements(code)?;
@@ -571,37 +576,38 @@ impl SymbolTableBuilder {
                 for name in names {
                     if let Some(alias) = &name.alias {
                         // `import mymodule as myalias`
-                        self.register_name(alias, SymbolUsage::Imported)?;
+                        self.register_name(alias, SymbolUsage::Imported, location)?;
                     } else {
                         // `import module`
                         self.register_name(
                             name.symbol.split('.').next().unwrap(),
                             SymbolUsage::Imported,
+                            location,
                         )?;
                     }
                 }
             }
             Return { value } => {
                 if let Some(expression) = value {
-                    self.scan_expression(expression, &ExpressionContext::Load)?;
+                    self.scan_expression(expression, ExpressionContext::Load)?;
                 }
             }
             Assert { test, msg } => {
-                self.scan_expression(test, &ExpressionContext::Load)?;
+                self.scan_expression(test, ExpressionContext::Load)?;
                 if let Some(expression) = msg {
-                    self.scan_expression(expression, &ExpressionContext::Load)?;
+                    self.scan_expression(expression, ExpressionContext::Load)?;
                 }
             }
             Delete { targets } => {
-                self.scan_expressions(targets, &ExpressionContext::Delete)?;
+                self.scan_expressions(targets, ExpressionContext::Delete)?;
             }
             Assign { targets, value } => {
-                self.scan_expressions(targets, &ExpressionContext::Store)?;
-                self.scan_expression(value, &ExpressionContext::Load)?;
+                self.scan_expressions(targets, ExpressionContext::Store)?;
+                self.scan_expression(value, ExpressionContext::Load)?;
             }
             AugAssign { target, value, .. } => {
-                self.scan_expression(target, &ExpressionContext::Store)?;
-                self.scan_expression(value, &ExpressionContext::Load)?;
+                self.scan_expression(target, ExpressionContext::Store)?;
+                self.scan_expression(value, ExpressionContext::Load)?;
             }
             AnnAssign {
                 target,
@@ -610,20 +616,20 @@ impl SymbolTableBuilder {
             } => {
                 // https://github.com/python/cpython/blob/master/Python/symtable.c#L1233
                 if let ast::ExpressionType::Identifier { ref name } = target.node {
-                    self.register_name(name, SymbolUsage::AnnotationAssigned)?;
+                    self.register_name(name, SymbolUsage::AnnotationAssigned, location)?;
                 } else {
-                    self.scan_expression(target, &ExpressionContext::Store)?;
+                    self.scan_expression(target, ExpressionContext::Store)?;
                 }
-                self.scan_expression(annotation, &ExpressionContext::Load)?;
+                self.scan_expression(annotation, ExpressionContext::Load)?;
                 if let Some(value) = value {
-                    self.scan_expression(value, &ExpressionContext::Load)?;
+                    self.scan_expression(value, ExpressionContext::Load)?;
                 }
             }
             With { items, body, .. } => {
                 for item in items {
-                    self.scan_expression(&item.context_expr, &ExpressionContext::Load)?;
+                    self.scan_expression(&item.context_expr, ExpressionContext::Load)?;
                     if let Some(expression) = &item.optional_vars {
-                        self.scan_expression(expression, &ExpressionContext::Store)?;
+                        self.scan_expression(expression, ExpressionContext::Store)?;
                     }
                 }
                 self.scan_statements(body)?;
@@ -637,10 +643,10 @@ impl SymbolTableBuilder {
                 self.scan_statements(body)?;
                 for handler in handlers {
                     if let Some(expression) = &handler.typ {
-                        self.scan_expression(expression, &ExpressionContext::Load)?;
+                        self.scan_expression(expression, ExpressionContext::Load)?;
                     }
                     if let Some(name) = &handler.name {
-                        self.register_name(name, SymbolUsage::Assigned)?;
+                        self.register_name(name, SymbolUsage::Assigned, location)?;
                     }
                     self.scan_statements(&handler.body)?;
                 }
@@ -653,10 +659,10 @@ impl SymbolTableBuilder {
             }
             Raise { exception, cause } => {
                 if let Some(expression) = exception {
-                    self.scan_expression(expression, &ExpressionContext::Load)?;
+                    self.scan_expression(expression, ExpressionContext::Load)?;
                 }
                 if let Some(expression) = cause {
-                    self.scan_expression(expression, &ExpressionContext::Load)?;
+                    self.scan_expression(expression, ExpressionContext::Load)?;
                 }
             }
         }
@@ -666,7 +672,7 @@ impl SymbolTableBuilder {
     fn scan_expressions(
         &mut self,
         expressions: &[ast::Expression],
-        context: &ExpressionContext,
+        context: ExpressionContext,
     ) -> SymbolTableResult {
         for expression in expressions {
             self.scan_expression(expression, context)?;
@@ -677,9 +683,10 @@ impl SymbolTableBuilder {
     fn scan_expression(
         &mut self,
         expression: &ast::Expression,
-        context: &ExpressionContext,
+        context: ExpressionContext,
     ) -> SymbolTableResult {
         use ast::ExpressionType::*;
+        let location = expression.location;
         match &expression.node {
             Binop { a, b, .. } => {
                 self.scan_expression(a, context)?;
@@ -692,11 +699,11 @@ impl SymbolTableBuilder {
                 self.scan_expressions(vals, context)?;
             }
             Subscript { a, b } => {
-                self.scan_expression(a, &ExpressionContext::Load)?;
-                self.scan_expression(b, &ExpressionContext::Load)?;
+                self.scan_expression(a, ExpressionContext::Load)?;
+                self.scan_expression(b, ExpressionContext::Load)?;
             }
             Attribute { value, .. } => {
-                self.scan_expression(value, &ExpressionContext::Load)?;
+                self.scan_expression(value, ExpressionContext::Load)?;
             }
             Dict { elements } => {
                 for (key, value) in elements {
@@ -740,41 +747,37 @@ impl SymbolTableBuilder {
                     ast::ComprehensionKind::Dict { .. } => "dictcomp",
                 };
 
-                self.enter_scope(
-                    scope_name,
-                    SymbolTableType::Comprehension,
-                    expression.location.row(),
-                );
+                self.enter_scope(scope_name, SymbolTableType::Comprehension, location.row());
 
                 // Register the passed argument to the generator function as the name ".0"
-                self.register_name(".0", SymbolUsage::Parameter)?;
+                self.register_name(".0", SymbolUsage::Parameter, location)?;
 
                 match **kind {
                     ast::ComprehensionKind::GeneratorExpression { ref element }
                     | ast::ComprehensionKind::List { ref element }
                     | ast::ComprehensionKind::Set { ref element } => {
-                        self.scan_expression(element, &ExpressionContext::Load)?;
+                        self.scan_expression(element, ExpressionContext::Load)?;
                     }
                     ast::ComprehensionKind::Dict { ref key, ref value } => {
-                        self.scan_expression(&key, &ExpressionContext::Load)?;
-                        self.scan_expression(&value, &ExpressionContext::Load)?;
+                        self.scan_expression(&key, ExpressionContext::Load)?;
+                        self.scan_expression(&value, ExpressionContext::Load)?;
                     }
                 }
 
                 let mut is_first_generator = true;
                 for generator in generators {
-                    self.scan_expression(&generator.target, &ExpressionContext::Iter)?;
+                    self.scan_expression(&generator.target, ExpressionContext::Iter)?;
                     if is_first_generator {
                         is_first_generator = false;
                     } else {
                         self.scan_expression(
                             &generator.iter,
-                            &ExpressionContext::IterDefinitionExp,
+                            ExpressionContext::IterDefinitionExp,
                         )?;
                     }
 
                     for if_expr in &generator.ifs {
-                        self.scan_expression(if_expr, &ExpressionContext::Load)?;
+                        self.scan_expression(if_expr, ExpressionContext::Load)?;
                     }
                 }
 
@@ -782,25 +785,25 @@ impl SymbolTableBuilder {
 
                 // The first iterable is passed as an argument into the created function:
                 assert!(!generators.is_empty());
-                self.scan_expression(&generators[0].iter, &ExpressionContext::IterDefinitionExp)?;
+                self.scan_expression(&generators[0].iter, ExpressionContext::IterDefinitionExp)?;
             }
             Call {
                 function,
                 args,
                 keywords,
             } => {
-                match *context {
+                match context {
                     ExpressionContext::IterDefinitionExp => {
-                        self.scan_expression(function, &ExpressionContext::IterDefinitionExp)?;
+                        self.scan_expression(function, ExpressionContext::IterDefinitionExp)?;
                     }
                     _ => {
-                        self.scan_expression(function, &ExpressionContext::Load)?;
+                        self.scan_expression(function, ExpressionContext::Load)?;
                     }
                 }
 
-                self.scan_expressions(args, &ExpressionContext::Load)?;
+                self.scan_expressions(args, ExpressionContext::Load)?;
                 for keyword in keywords {
-                    self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
+                    self.scan_expression(&keyword.value, ExpressionContext::Load)?;
                 }
             }
             String { value } => {
@@ -810,48 +813,49 @@ impl SymbolTableBuilder {
                 // Determine the contextual usage of this symbol:
                 match context {
                     ExpressionContext::Delete => {
-                        self.register_name(name, SymbolUsage::Used)?;
+                        self.register_name(name, SymbolUsage::Used, location)?;
                     }
                     ExpressionContext::Load | ExpressionContext::IterDefinitionExp => {
-                        self.register_name(name, SymbolUsage::Used)?;
+                        self.register_name(name, SymbolUsage::Used, location)?;
                     }
                     ExpressionContext::Store => {
-                        self.register_name(name, SymbolUsage::Assigned)?;
+                        self.register_name(name, SymbolUsage::Assigned, location)?;
                     }
                     ExpressionContext::Iter => {
-                        self.register_name(name, SymbolUsage::Iter)?;
+                        self.register_name(name, SymbolUsage::Iter, location)?;
                     }
                 }
             }
             Lambda { args, body } => {
                 self.enter_function("lambda", args, expression.location.row())?;
-                match *context {
+                match context {
                     ExpressionContext::IterDefinitionExp => {
-                        self.scan_expression(body, &ExpressionContext::IterDefinitionExp)?;
+                        self.scan_expression(body, ExpressionContext::IterDefinitionExp)?;
                     }
                     _ => {
-                        self.scan_expression(body, &ExpressionContext::Load)?;
+                        self.scan_expression(body, ExpressionContext::Load)?;
                     }
                 }
                 self.leave_scope();
             }
             IfExpression { test, body, orelse } => {
-                self.scan_expression(test, &ExpressionContext::Load)?;
-                self.scan_expression(body, &ExpressionContext::Load)?;
-                self.scan_expression(orelse, &ExpressionContext::Load)?;
+                self.scan_expression(test, ExpressionContext::Load)?;
+                self.scan_expression(body, ExpressionContext::Load)?;
+                self.scan_expression(orelse, ExpressionContext::Load)?;
             }
 
             NamedExpression { left, right } => {
                 // named expressions are not allowed in the definiton of
                 // comprehension iterator definitions
-                if let ExpressionContext::IterDefinitionExp = *context {
+                if let ExpressionContext::IterDefinitionExp = context {
                     return Err(SymbolTableError {
                         error: "assignment expression cannot be used in a comprehension iterable expression".to_string(),
-                        location: Default::default(),
+                        // TODO: accurate location info, somehow
+                        location: Location::default(),
                     });
                 }
 
-                self.scan_expression(right, &ExpressionContext::Load)?;
+                self.scan_expression(right, ExpressionContext::Load)?;
 
                 // special handling for assigned identifier in named expressions
                 // that are used in comprehensions. This required to correctly
@@ -860,15 +864,15 @@ impl SymbolTableBuilder {
                 if let Identifier { name } = &left.node {
                     let table = self.tables.last().unwrap();
                     if table.typ == SymbolTableType::Comprehension {
-                        self.register_name(name, SymbolUsage::AssignedNamedExprInCompr)?;
+                        self.register_name(name, SymbolUsage::AssignedNamedExprInCompr, location)?;
                     } else {
                         // omit one recursion. When the handling of an store changes for
                         // Identifiers this needs adapted - more forward safe would be
                         // calling scan_expression directly.
-                        self.register_name(name, SymbolUsage::Assigned)?;
+                        self.register_name(name, SymbolUsage::Assigned, location)?;
                     }
                 } else {
-                    self.scan_expression(left, &ExpressionContext::Store)?;
+                    self.scan_expression(left, ExpressionContext::Store)?;
                 }
             }
         }
@@ -882,10 +886,10 @@ impl SymbolTableBuilder {
         line_number: usize,
     ) -> SymbolTableResult {
         // Evaluate eventual default parameters:
-        self.scan_expressions(&args.defaults, &ExpressionContext::Load)?;
+        self.scan_expressions(&args.defaults, ExpressionContext::Load)?;
         for kw_default in &args.kw_defaults {
             if let Some(expression) = kw_default {
-                self.scan_expression(&expression, &ExpressionContext::Load)?;
+                self.scan_expression(&expression, ExpressionContext::Load)?;
             }
         }
 
@@ -917,7 +921,7 @@ impl SymbolTableBuilder {
         match group {
             ast::StringGroup::Constant { .. } => {}
             ast::StringGroup::FormattedValue { value, spec, .. } => {
-                self.scan_expression(value, &ExpressionContext::Load)?;
+                self.scan_expression(value, ExpressionContext::Load)?;
                 if let Some(spec) = spec {
                     self.scan_string_group(spec)?;
                 }
@@ -931,10 +935,14 @@ impl SymbolTableBuilder {
         Ok(())
     }
 
-    fn register_name(&mut self, name: &str, role: SymbolUsage) -> SymbolTableResult {
+    fn register_name(
+        &mut self,
+        name: &str,
+        role: SymbolUsage,
+        location: Location,
+    ) -> SymbolTableResult {
         let scope_depth = self.tables.len();
         let table = self.tables.last_mut().unwrap();
-        let location = Default::default();
 
         // Some checks for the symbol that present on this scope level:
         if let Some(symbol) = table.symbols.get(name) {

From 31b884d37a55054544e1496e4d16dbb773e2e7d8 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 2 Nov 2020 23:15:56 -0600
Subject: [PATCH 163/459] compiler/porcelain wrapper

---
 Cargo.toml           |   2 +-
 porcelain/Cargo.toml |  12 ++++
 porcelain/src/lib.rs | 135 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 148 insertions(+), 1 deletion(-)
 create mode 100644 porcelain/Cargo.toml
 create mode 100644 porcelain/src/lib.rs

diff --git a/Cargo.toml b/Cargo.toml
index dd9f1d0b..dc44869a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "rustpython-compiler"
+name = "rustpython-compiler-core"
 version = "0.1.2"
 description = "Compiler for python code into bytecode for the rustpython VM."
 authors = ["RustPython Team"]
diff --git a/porcelain/Cargo.toml b/porcelain/Cargo.toml
new file mode 100644
index 00000000..e07c746c
--- /dev/null
+++ b/porcelain/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "rustpython-compiler"
+version = "0.1.2"
+description = "A usability wrapper around rustpython-parser and rustpython-compiler-core"
+authors = ["RustPython Team"]
+edition = "2018"
+
+[dependencies]
+thiserror = "1.0"
+rustpython-compiler-core = { path = ".." }
+rustpython-parser = { path = "../../parser" }
+rustpython-bytecode = { path = "../../bytecode" }
diff --git a/porcelain/src/lib.rs b/porcelain/src/lib.rs
new file mode 100644
index 00000000..bdf93ee0
--- /dev/null
+++ b/porcelain/src/lib.rs
@@ -0,0 +1,135 @@
+use rustpython_bytecode::bytecode::CodeObject;
+use rustpython_compiler_core::{compile, symboltable};
+use rustpython_parser::{ast::Location, parser};
+use std::fmt;
+
+pub use compile::{CompileOpts, Mode};
+pub use symboltable::{Symbol, SymbolScope, SymbolTable, SymbolTableType};
+
+#[derive(Debug, thiserror::Error)]
+pub enum CompileErrorType {
+    #[error(transparent)]
+    Compile(#[from] rustpython_compiler_core::error::CompileErrorType),
+    #[error(transparent)]
+    Parse(#[from] rustpython_parser::error::ParseErrorType),
+}
+
+#[derive(Debug, thiserror::Error)]
+pub struct CompileError {
+    pub error: CompileErrorType,
+    pub statement: Option<String>,
+    pub source_path: String,
+    pub location: Location,
+}
+
+impl fmt::Display for CompileError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let loc = self.location;
+        if let Some(ref stmt) = self.statement {
+            // visualize the error when location and statement are provided
+            write!(
+                f,
+                "{}",
+                loc.visualize(stmt, &format_args!("{} at {}", self.error, loc))
+            )
+        } else {
+            write!(f, "{} at {}", self.error, loc)
+        }
+    }
+}
+
+impl CompileError {
+    fn from_compile(error: rustpython_compiler_core::error::CompileError, source: &str) -> Self {
+        CompileError {
+            error: error.error.into(),
+            location: error.location,
+            source_path: error.source_path,
+            statement: get_statement(source, error.location),
+        }
+    }
+    fn from_parse(
+        error: rustpython_parser::error::ParseError,
+        source: &str,
+        source_path: String,
+    ) -> Self {
+        CompileError {
+            error: error.error.into(),
+            location: error.location,
+            source_path,
+            statement: get_statement(source, error.location),
+        }
+    }
+    fn from_symtable(
+        error: symboltable::SymbolTableError,
+        source: &str,
+        source_path: String,
+    ) -> Self {
+        Self::from_compile(error.into_compile_error(source_path), source)
+    }
+}
+
+/// Compile a given sourcecode into a bytecode object.
+pub fn compile(
+    source: &str,
+    mode: compile::Mode,
+    source_path: String,
+    opts: CompileOpts,
+) -> Result<CodeObject, CompileError> {
+    macro_rules! try_parse {
+        ($x:expr) => {
+            match $x {
+                Ok(x) => x,
+                Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
+            }
+        };
+    }
+    let res = match mode {
+        compile::Mode::Exec => {
+            let ast = try_parse!(parser::parse_program(source));
+            compile::compile_program(ast, source_path, opts)
+        }
+        compile::Mode::Eval => {
+            let statement = try_parse!(parser::parse_statement(source));
+            compile::compile_statement_eval(statement, source_path, opts)
+        }
+        compile::Mode::Single => {
+            let ast = try_parse!(parser::parse_program(source));
+            compile::compile_program_single(ast, source_path, opts)
+        }
+    };
+    res.map_err(|e| CompileError::from_compile(e, source))
+}
+
+pub fn compile_symtable(
+    source: &str,
+    mode: compile::Mode,
+    source_path: &str,
+) -> Result<symboltable::SymbolTable, CompileError> {
+    macro_rules! try_parse {
+        ($x:expr) => {
+            match $x {
+                Ok(x) => x,
+                Err(e) => return Err(CompileError::from_parse(e, source, source_path.to_owned())),
+            }
+        };
+    }
+    let res = match mode {
+        compile::Mode::Exec | compile::Mode::Single => {
+            let ast = try_parse!(parser::parse_program(source));
+            symboltable::make_symbol_table(&ast)
+        }
+        compile::Mode::Eval => {
+            let statement = try_parse!(parser::parse_statement(source));
+            symboltable::statements_to_symbol_table(&statement)
+        }
+    };
+    res.map_err(|e| CompileError::from_symtable(e, source, source_path.to_owned()))
+}
+
+fn get_statement(source: &str, loc: Location) -> Option<String> {
+    if loc.column() == 0 || loc.row() == 0 {
+        return None;
+    }
+    let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
+    Some(line + "\n")
+}

From e37a55e74cc64b8483e139fd4ef7c289a5f83316 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sun, 8 Nov 2020 20:46:24 -0600
Subject: [PATCH 164/459] Implement string interning for variable names

---
 src/compile.rs | 174 ++++++++++++++++++++++++++-----------------------
 1 file changed, 94 insertions(+), 80 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 6c371af5..7a6bf0f8 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -10,6 +10,7 @@ pub use crate::mode::Mode;
 use crate::symboltable::{
     make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope, SymbolTable,
 };
+use indexmap::IndexSet;
 use itertools::Itertools;
 use num_complex::Complex64;
 use rustpython_ast as ast;
@@ -19,7 +20,7 @@ type CompileResult<T> = Result<T, CompileError>;
 
 /// Main structure holding the state of compilation.
 struct Compiler {
-    output_stack: Vec<CodeObject>,
+    output_stack: Vec<(CodeObject, IndexSet<String>)>,
     symbol_table_stack: Vec<SymbolTable>,
     nxt_label: usize,
     source_path: String,
@@ -150,7 +151,7 @@ impl Compiler {
     }
 
     fn push_output(&mut self, code: CodeObject) {
-        self.output_stack.push(code);
+        self.output_stack.push((code, IndexSet::new()));
     }
 
     fn push_new_code_object(&mut self, obj_name: String) {
@@ -158,10 +159,8 @@ impl Compiler {
         self.push_output(CodeObject::new(
             Default::default(),
             0,
-            Vec::new(),
-            None,
-            Vec::new(),
-            None,
+            0,
+            0,
             self.source_path.clone(),
             line_number,
             obj_name,
@@ -169,7 +168,20 @@ impl Compiler {
     }
 
     fn pop_code_object(&mut self) -> CodeObject {
-        self.output_stack.pop().unwrap()
+        let (mut code, names) = self.output_stack.pop().unwrap();
+        code.names.extend(names);
+        code
+    }
+
+    // could take impl Into<Cow<str>>, but everything is borrowed from ast structs; we never
+    // actually have a `String` to pass
+    fn name(&mut self, name: &str) -> bytecode::NameIdx {
+        let cache = &mut self.output_stack.last_mut().expect("nothing on stack").1;
+        if let Some(x) = cache.get_index_of(name) {
+            x
+        } else {
+            cache.insert_full(name.to_owned()).0
+        }
     }
 
     fn compile_program(
@@ -183,8 +195,9 @@ impl Compiler {
         let (statements, doc) = get_doc(&program.statements);
         if let Some(value) = doc {
             self.emit_constant(bytecode::ConstantData::Str { value });
+            let doc = self.name("__doc__");
             self.emit(Instruction::StoreName {
-                name: "__doc__".to_owned(),
+                idx: doc,
                 scope: bytecode::NameScope::Global,
             });
         }
@@ -279,18 +292,14 @@ impl Compiler {
 
     fn load_name(&mut self, name: &str) {
         let scope = self.scope_for_name(name);
-        self.emit(Instruction::LoadName {
-            name: name.to_owned(),
-            scope,
-        });
+        let idx = self.name(name);
+        self.emit(Instruction::LoadName { idx, scope });
     }
 
     fn store_name(&mut self, name: &str) {
         let scope = self.scope_for_name(name);
-        self.emit(Instruction::StoreName {
-            name: name.to_owned(),
-            scope,
-        });
+        let idx = self.name(name);
+        self.emit(Instruction::StoreName { idx, scope });
     }
 
     fn compile_statement(&mut self, statement: &ast::Statement) -> CompileResult<()> {
@@ -312,16 +321,16 @@ impl Compiler {
             Import { names } => {
                 // import a, b, c as d
                 for name in names {
+                    let name_idx = Some(self.name(&name.symbol));
                     self.emit(Instruction::Import {
-                        name: Some(name.symbol.clone()),
-                        symbols: vec![],
+                        name_idx,
+                        symbols_idx: vec![],
                         level: 0,
                     });
                     if let Some(alias) = &name.alias {
                         for part in name.symbol.split('.').skip(1) {
-                            self.emit(Instruction::LoadAttr {
-                                name: part.to_owned(),
-                            });
+                            let idx = self.name(part);
+                            self.emit(Instruction::LoadAttr { idx });
                         }
                         self.store_name(alias);
                     } else {
@@ -336,31 +345,33 @@ impl Compiler {
             } => {
                 let import_star = names.iter().any(|n| n.symbol == "*");
 
+                let module_idx = module.as_ref().map(|s| self.name(s));
+
                 if import_star {
+                    let star = self.name("*");
                     // from .... import *
                     self.emit(Instruction::Import {
-                        name: module.clone(),
-                        symbols: vec!["*".to_owned()],
+                        name_idx: module_idx,
+                        symbols_idx: vec![star],
                         level: *level,
                     });
                     self.emit(Instruction::ImportStar);
                 } else {
                     // from mod import a, b as c
                     // First, determine the fromlist (for import lib):
-                    let from_list = names.iter().map(|n| n.symbol.clone()).collect();
+                    let from_list = names.iter().map(|n| self.name(&n.symbol)).collect();
 
                     // Load module once:
                     self.emit(Instruction::Import {
-                        name: module.clone(),
-                        symbols: from_list,
+                        name_idx: module_idx,
+                        symbols_idx: from_list,
                         level: *level,
                     });
 
                     for name in names {
+                        let idx = self.name(&name.symbol);
                         // import symbol from module:
-                        self.emit(Instruction::ImportFrom {
-                            name: name.symbol.to_owned(),
-                        });
+                        self.emit(Instruction::ImportFrom { idx });
 
                         // Store module under proper name:
                         if let Some(alias) = &name.alias {
@@ -513,8 +524,9 @@ impl Compiler {
                 if self.opts.optimize == 0 {
                     let end_label = self.new_label();
                     self.compile_jump_if(test, true, end_label)?;
+                    let assertion_error = self.name("AssertionError");
                     self.emit(Instruction::LoadName {
-                        name: String::from("AssertionError"),
+                        idx: assertion_error,
                         scope: bytecode::NameScope::Global,
                     });
                     match msg {
@@ -612,15 +624,13 @@ impl Compiler {
     fn compile_delete(&mut self, expression: &ast::Expression) -> CompileResult<()> {
         match &expression.node {
             ast::ExpressionType::Identifier { name } => {
-                self.emit(Instruction::DeleteName {
-                    name: name.to_owned(),
-                });
+                let idx = self.name(name);
+                self.emit(Instruction::DeleteName { idx });
             }
             ast::ExpressionType::Attribute { value, name } => {
                 self.compile_expression(value)?;
-                self.emit(Instruction::DeleteAttr {
-                    name: name.to_owned(),
-                });
+                let idx = self.name(name);
+                self.emit(Instruction::DeleteAttr { idx });
             }
             ast::ExpressionType::Subscript { a, b } => {
                 self.compile_expression(a)?;
@@ -677,33 +687,35 @@ impl Compiler {
             flags |= bytecode::CodeFlags::HAS_KW_ONLY_DEFAULTS;
         }
 
-        let mut compile_varargs = |va: &ast::Varargs, flag| match va {
-            ast::Varargs::None => None,
-            ast::Varargs::Unnamed => {
-                flags |= flag;
-                None
-            }
-            ast::Varargs::Named(name) => {
-                flags |= flag;
-                Some(name.arg.clone())
-            }
-        };
-
-        let varargs_name = compile_varargs(&args.vararg, bytecode::CodeFlags::HAS_VARARGS);
-        let varkeywords_name = compile_varargs(&args.kwarg, bytecode::CodeFlags::HAS_VARKEYWORDS);
-
         let line_number = self.get_source_line_number();
         self.push_output(CodeObject::new(
             flags,
             args.posonlyargs_count,
-            args.args.iter().map(|a| a.arg.clone()).collect(),
-            varargs_name,
-            args.kwonlyargs.iter().map(|a| a.arg.clone()).collect(),
-            varkeywords_name,
+            args.args.len(),
+            args.kwonlyargs.len(),
             self.source_path.clone(),
             line_number,
             name.to_owned(),
         ));
+
+        for name in &args.args {
+            self.name(&name.arg);
+        }
+        for name in &args.kwonlyargs {
+            self.name(&name.arg);
+        }
+
+        let mut compile_varargs = |va: &ast::Varargs, flag| match va {
+            ast::Varargs::None | ast::Varargs::Unnamed => {}
+            ast::Varargs::Named(name) => {
+                self.current_code().flags |= flag;
+                self.name(&name.arg);
+            }
+        };
+
+        compile_varargs(&args.vararg, bytecode::CodeFlags::HAS_VARARGS);
+        compile_varargs(&args.kwarg, bytecode::CodeFlags::HAS_VARKEYWORDS);
+
         self.enter_scope();
 
         Ok(())
@@ -948,9 +960,8 @@ impl Compiler {
         self.emit(Instruction::Duplicate);
         self.load_docstring(doc_str);
         self.emit(Instruction::Rotate { amount: 2 });
-        self.emit(Instruction::StoreAttr {
-            name: "__doc__".to_owned(),
-        });
+        let doc = self.name("__doc__");
+        self.emit(Instruction::StoreAttr { idx: doc });
         self.apply_decorators(decorator_list);
 
         self.store_name(name);
@@ -1041,10 +1052,8 @@ impl Compiler {
         self.push_output(CodeObject::new(
             Default::default(),
             0,
-            vec![],
-            None,
-            vec![],
-            None,
+            0,
+            0,
             self.source_path.clone(),
             line_number,
             name.to_owned(),
@@ -1053,24 +1062,28 @@ impl Compiler {
 
         let (new_body, doc_str) = get_doc(body);
 
+        let dunder_name = self.name("__name__");
         self.emit(Instruction::LoadName {
-            name: "__name__".to_owned(),
+            idx: dunder_name,
             scope: bytecode::NameScope::Global,
         });
+        let dunder_module = self.name("__module__");
         self.emit(Instruction::StoreName {
-            name: "__module__".to_owned(),
+            idx: dunder_module,
             scope: bytecode::NameScope::Free,
         });
         self.emit_constant(bytecode::ConstantData::Str {
             value: qualified_name.clone(),
         });
+        let qualname = self.name("__qualname__");
         self.emit(Instruction::StoreName {
-            name: "__qualname__".to_owned(),
+            idx: qualname,
             scope: bytecode::NameScope::Free,
         });
         self.load_docstring(doc_str);
+        let doc = self.name("__doc__");
         self.emit(Instruction::StoreName {
-            name: "__doc__".to_owned(),
+            idx: doc,
             scope: bytecode::NameScope::Free,
         });
         // setup annotations
@@ -1222,8 +1235,9 @@ impl Compiler {
 
             self.set_label(check_asynciter_label);
             self.emit(Instruction::Duplicate);
+            let stopasynciter = self.name("StopAsyncIteration");
             self.emit(Instruction::LoadName {
-                name: "StopAsyncIteration".to_owned(),
+                idx: stopasynciter,
                 scope: bytecode::NameScope::Global,
             });
             self.emit(Instruction::CompareOperation {
@@ -1363,8 +1377,9 @@ impl Compiler {
 
         if let ast::ExpressionType::Identifier { name } = &target.node {
             // Store as dict entry in __annotations__ dict:
+            let annotations = self.name("__annotations__");
             self.emit(Instruction::LoadName {
-                name: String::from("__annotations__"),
+                idx: annotations,
                 scope: bytecode::NameScope::Local,
             });
             self.emit_constant(bytecode::ConstantData::Str {
@@ -1390,9 +1405,8 @@ impl Compiler {
             }
             ast::ExpressionType::Attribute { value, name } => {
                 self.compile_expression(value)?;
-                self.emit(Instruction::StoreAttr {
-                    name: name.to_owned(),
-                });
+                let idx = self.name(name);
+                self.emit(Instruction::StoreAttr { idx });
             }
             ast::ExpressionType::List { elements } | ast::ExpressionType::Tuple { elements } => {
                 let mut seen_star = false;
@@ -1658,9 +1672,8 @@ impl Compiler {
             }
             Attribute { value, name } => {
                 self.compile_expression(value)?;
-                self.emit(Instruction::LoadAttr {
-                    name: name.to_owned(),
-                });
+                let idx = self.name(name);
+                self.emit(Instruction::LoadAttr { idx });
             }
             Compare { vals, ops } => {
                 self.compile_chained_comparison(vals, ops)?;
@@ -1966,14 +1979,13 @@ impl Compiler {
         self.push_output(CodeObject::new(
             Default::default(),
             1,
-            vec![".0".to_owned()],
-            None,
-            vec![],
-            None,
+            1,
+            0,
             self.source_path.clone(),
             line_number,
             name.clone(),
         ));
+        let arg0 = self.name(".0");
         self.enter_scope();
 
         // Create empty object of proper type:
@@ -2009,7 +2021,7 @@ impl Compiler {
             if loop_labels.is_empty() {
                 // Load iterator onto stack (passed as first argument):
                 self.emit(Instruction::LoadName {
-                    name: String::from(".0"),
+                    idx: arg0,
                     scope: bytecode::NameScope::Local,
                 });
             } else {
@@ -2228,9 +2240,11 @@ impl Compiler {
     }
 
     fn current_code(&mut self) -> &mut CodeObject {
-        self.output_stack
+        &mut self
+            .output_stack
             .last_mut()
             .expect("No OutputStream on stack")
+            .0
     }
 
     // Generate a new label

From 38c4c14ee359dd28ddb185c3d569944bf5960c1c Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 23 Nov 2020 14:39:56 -0600
Subject: [PATCH 165/459] Remove the need for a label_map in CodeObject

---
 src/compile.rs | 91 +++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 72 insertions(+), 19 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 7a6bf0f8..b51d6633 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -18,11 +18,52 @@ use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Lab
 
 type CompileResult<T> = Result<T, CompileError>;
 
+struct CodeInfo {
+    code: CodeObject,
+    name_cache: IndexSet<String>,
+    label_map: Vec<Option<Label>>,
+}
+impl CodeInfo {
+    fn finalize_code(self) -> CodeObject {
+        let CodeInfo {
+            mut code,
+            name_cache,
+            label_map,
+        } = self;
+        code.names.extend(name_cache);
+        for instruction in &mut code.instructions {
+            use Instruction::*;
+            // this is a little bit hacky, as until now the data stored inside Labels in
+            // Instructions is just bookkeeping, but I think it's the best way to do this
+            // XXX: any new instruction that uses a label has to be added here
+            let label = match instruction {
+                Jump { target } => target,
+                JumpIfTrue { target } => target,
+                JumpIfFalse { target } => target,
+                JumpIfTrueOrPop { target } => target,
+                JumpIfFalseOrPop { target } => target,
+                ForIter { target } => target,
+                SetupLoop { start, end } => {
+                    *start = label_map[start.0].expect("label never set");
+                    *end = label_map[end.0].expect("label never set");
+                    continue;
+                }
+                SetupFinally { handler } => handler,
+                SetupExcept { handler } => handler,
+                SetupWith { end } => end,
+                SetupAsyncWith { end } => end,
+                _ => continue,
+            };
+            *label = label_map[label.0].expect("label never set");
+        }
+        code
+    }
+}
+
 /// Main structure holding the state of compilation.
 struct Compiler {
-    output_stack: Vec<(CodeObject, IndexSet<String>)>,
+    code_stack: Vec<CodeInfo>,
     symbol_table_stack: Vec<SymbolTable>,
-    nxt_label: usize,
     source_path: String,
     current_source_location: ast::Location,
     current_qualified_path: Option<String>,
@@ -124,9 +165,8 @@ pub fn compile_program_single(
 impl Compiler {
     fn new(opts: CompileOpts, source_path: String) -> Self {
         Compiler {
-            output_stack: Vec::new(),
+            code_stack: Vec::new(),
             symbol_table_stack: Vec::new(),
-            nxt_label: 0,
             source_path,
             current_source_location: ast::Location::default(),
             current_qualified_path: None,
@@ -151,7 +191,11 @@ impl Compiler {
     }
 
     fn push_output(&mut self, code: CodeObject) {
-        self.output_stack.push((code, IndexSet::new()));
+        self.code_stack.push(CodeInfo {
+            code,
+            name_cache: IndexSet::new(),
+            label_map: Vec::new(),
+        });
     }
 
     fn push_new_code_object(&mut self, obj_name: String) {
@@ -168,15 +212,17 @@ impl Compiler {
     }
 
     fn pop_code_object(&mut self) -> CodeObject {
-        let (mut code, names) = self.output_stack.pop().unwrap();
-        code.names.extend(names);
-        code
+        self.code_stack.pop().unwrap().finalize_code()
     }
 
     // could take impl Into<Cow<str>>, but everything is borrowed from ast structs; we never
     // actually have a `String` to pass
     fn name(&mut self, name: &str) -> bytecode::NameIdx {
-        let cache = &mut self.output_stack.last_mut().expect("nothing on stack").1;
+        let cache = &mut self
+            .code_stack
+            .last_mut()
+            .expect("nothing on stack")
+            .name_cache;
         if let Some(x) = cache.get_index_of(name) {
             x
         } else {
@@ -189,7 +235,7 @@ impl Compiler {
         program: &ast::Program,
         symbol_table: SymbolTable,
     ) -> CompileResult<()> {
-        let size_before = self.output_stack.len();
+        let size_before = self.code_stack.len();
         self.symbol_table_stack.push(symbol_table);
 
         let (statements, doc) = get_doc(&program.statements);
@@ -203,7 +249,7 @@ impl Compiler {
         }
         self.compile_statements(statements)?;
 
-        assert_eq!(self.output_stack.len(), size_before);
+        assert_eq!(self.code_stack.len(), size_before);
 
         // Emit None at end:
         self.emit_constant(bytecode::ConstantData::None);
@@ -2241,24 +2287,31 @@ impl Compiler {
 
     fn current_code(&mut self) -> &mut CodeObject {
         &mut self
-            .output_stack
+            .code_stack
             .last_mut()
             .expect("No OutputStream on stack")
-            .0
+            .code
     }
 
     // Generate a new label
     fn new_label(&mut self) -> Label {
-        let l = Label::new(self.nxt_label);
-        self.nxt_label += 1;
-        l
+        let label_map = &mut self.code_stack.last_mut().unwrap().label_map;
+        let label = Label(label_map.len());
+        label_map.push(None);
+        label
     }
 
     // Assign current position the given label
     fn set_label(&mut self, label: Label) {
-        let code = self.current_code();
-        let pos = code.instructions.len();
-        code.label_map.insert(label, pos);
+        let CodeInfo {
+            code, label_map, ..
+        } = self.code_stack.last_mut().unwrap();
+        let actual_label = Label(code.instructions.len());
+        let prev_val = std::mem::replace(&mut label_map[label.0], Some(actual_label));
+        debug_assert!(
+            prev_val.map_or(true, |x| x == actual_label),
+            "double-set a label"
+        );
     }
 
     fn set_source_location(&mut self, location: ast::Location) {

From df41efed8a939ac1daf3c7abf05c4bf657272c00 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Tue, 24 Nov 2020 11:57:14 -0600
Subject: [PATCH 166/459] Don't use a wildcard to prevent future mistakes

---
 src/compile.rs | 43 ++++++++++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 15 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index b51d6633..d0618830 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -36,25 +36,38 @@ impl CodeInfo {
             // this is a little bit hacky, as until now the data stored inside Labels in
             // Instructions is just bookkeeping, but I think it's the best way to do this
             // XXX: any new instruction that uses a label has to be added here
-            let label = match instruction {
-                Jump { target } => target,
-                JumpIfTrue { target } => target,
-                JumpIfFalse { target } => target,
-                JumpIfTrueOrPop { target } => target,
-                JumpIfFalseOrPop { target } => target,
-                ForIter { target } => target,
+            match instruction {
+                Jump { target: l }
+                | JumpIfTrue { target: l }
+                | JumpIfFalse { target: l }
+                | JumpIfTrueOrPop { target: l }
+                | JumpIfFalseOrPop { target: l }
+                | ForIter { target: l }
+                | SetupFinally { handler: l }
+                | SetupExcept { handler: l }
+                | SetupWith { end: l }
+                | SetupAsyncWith { end: l } => {
+                    *l = label_map[l.0].expect("label never set");
+                }
                 SetupLoop { start, end } => {
                     *start = label_map[start.0].expect("label never set");
                     *end = label_map[end.0].expect("label never set");
-                    continue;
                 }
-                SetupFinally { handler } => handler,
-                SetupExcept { handler } => handler,
-                SetupWith { end } => end,
-                SetupAsyncWith { end } => end,
-                _ => continue,
-            };
-            *label = label_map[label.0].expect("label never set");
+
+                #[rustfmt::skip]
+                Import { .. } | ImportStar | ImportFrom { .. } | LoadName { .. } | StoreName { .. }
+                | DeleteName { .. } | Subscript | StoreSubscript | DeleteSubscript
+                | StoreAttr { .. } | DeleteAttr { .. } | LoadConst { .. } | UnaryOperation { .. }
+                | BinaryOperation { .. } | LoadAttr { .. } | CompareOperation { .. } | Pop
+                | Rotate { .. } | Duplicate | GetIter | Continue | Break | MakeFunction
+                | CallFunction { .. } | ReturnValue | YieldValue | YieldFrom | SetupAnnotation
+                | EnterFinally | EndFinally | WithCleanupStart | WithCleanupFinish | PopBlock
+                | Raise { .. } | BuildString { .. } | BuildTuple { .. } | BuildList { .. }
+                | BuildSet { .. } | BuildMap { .. } | BuildSlice { .. } | ListAppend { .. }
+                | SetAdd { .. } | MapAdd { .. } | PrintExpr | LoadBuildClass | UnpackSequence { .. }
+                | UnpackEx { .. } | FormatValue { .. } | PopException | Reverse { .. }
+                | GetAwaitable | BeforeAsyncWith | GetAIter | GetANext | MapAddRev { .. } => {}
+            }
         }
         code
     }

From 534ca2a8def324e7ea2db052f8f4ecd3b076e9b2 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 30 Nov 2020 22:16:23 -0600
Subject: [PATCH 167/459] Move compiler snapshots

---
 ...nap => rustpython_compiler_core__compile__tests__if_ands.snap} | 0
 ...ap => rustpython_compiler_core__compile__tests__if_mixed.snap} | 0
 ...snap => rustpython_compiler_core__compile__tests__if_ors.snap} | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 rename src/snapshots/{rustpython_compiler__compile__tests__if_ands.snap => rustpython_compiler_core__compile__tests__if_ands.snap} (100%)
 rename src/snapshots/{rustpython_compiler__compile__tests__if_mixed.snap => rustpython_compiler_core__compile__tests__if_mixed.snap} (100%)
 rename src/snapshots/{rustpython_compiler__compile__tests__if_ors.snap => rustpython_compiler_core__compile__tests__if_ors.snap} (100%)

diff --git a/src/snapshots/rustpython_compiler__compile__tests__if_ands.snap b/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap
similarity index 100%
rename from src/snapshots/rustpython_compiler__compile__tests__if_ands.snap
rename to src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap
diff --git a/src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap b/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap
similarity index 100%
rename from src/snapshots/rustpython_compiler__compile__tests__if_mixed.snap
rename to src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap
diff --git a/src/snapshots/rustpython_compiler__compile__tests__if_ors.snap b/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap
similarity index 100%
rename from src/snapshots/rustpython_compiler__compile__tests__if_ors.snap
rename to src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap

From 410bd76f38929fb30dd9706a90c40c40e49e20ce Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 28 Nov 2020 14:51:28 -0600
Subject: [PATCH 168/459] Fast locals part 1

---
 src/compile.rs     | 347 ++++++++++++++++++++++++++-------------------
 src/symboltable.rs |  85 ++++++-----
 2 files changed, 248 insertions(+), 184 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index d0618830..3137b47b 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -7,9 +7,7 @@
 
 use crate::error::{CompileError, CompileErrorType};
 pub use crate::mode::Mode;
-use crate::symboltable::{
-    make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope, SymbolTable,
-};
+use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolScope, SymbolTable};
 use indexmap::IndexSet;
 use itertools::Itertools;
 use num_complex::Complex64;
@@ -21,6 +19,9 @@ type CompileResult<T> = Result<T, CompileError>;
 struct CodeInfo {
     code: CodeObject,
     name_cache: IndexSet<String>,
+    varname_cache: IndexSet<String>,
+    cellvar_cache: IndexSet<String>,
+    freevar_cache: IndexSet<String>,
     label_map: Vec<Option<Label>>,
 }
 impl CodeInfo {
@@ -28,9 +29,17 @@ impl CodeInfo {
         let CodeInfo {
             mut code,
             name_cache,
+            varname_cache,
+            cellvar_cache,
+            freevar_cache,
             label_map,
         } = self;
+
         code.names.extend(name_cache);
+        code.varnames.extend(varname_cache);
+        code.cellvars.extend(cellvar_cache);
+        code.freevars.extend(freevar_cache);
+
         for instruction in &mut code.instructions {
             use Instruction::*;
             // this is a little bit hacky, as until now the data stored inside Labels in
@@ -55,8 +64,10 @@ impl CodeInfo {
                 }
 
                 #[rustfmt::skip]
-                Import { .. } | ImportStar | ImportFrom { .. } | LoadName { .. } | StoreName { .. }
-                | DeleteName { .. } | Subscript | StoreSubscript | DeleteSubscript
+                Import { .. } | ImportStar | ImportFrom { .. } | LoadFast(_) | LoadLocal(_)
+                | LoadGlobal(_) | LoadDeref(_) | LoadClassDeref(_) | StoreFast(_) | StoreLocal(_)
+                | StoreGlobal(_) | StoreDeref(_) | DeleteFast(_) | DeleteLocal(_) | DeleteGlobal(_)
+                | DeleteDeref(_) | LoadClosure(_) | Subscript | StoreSubscript | DeleteSubscript
                 | StoreAttr { .. } | DeleteAttr { .. } | LoadConst { .. } | UnaryOperation { .. }
                 | BinaryOperation { .. } | LoadAttr { .. } | CompareOperation { .. } | Pop
                 | Rotate { .. } | Duplicate | GetIter | Continue | Break | MakeFunction
@@ -73,6 +84,12 @@ impl CodeInfo {
     }
 }
 
+enum NameUsage {
+    Load,
+    Store,
+    Delete,
+}
+
 /// Main structure holding the state of compilation.
 struct Compiler {
     code_stack: Vec<CodeInfo>,
@@ -100,6 +117,7 @@ impl Default for CompileOpts {
 #[derive(Clone, Copy)]
 struct CompileContext {
     in_loop: bool,
+    in_class: bool,
     func: FunctionContext,
 }
 
@@ -122,8 +140,7 @@ fn with_compiler(
     opts: CompileOpts,
     f: impl FnOnce(&mut Compiler) -> CompileResult<()>,
 ) -> CompileResult<CodeObject> {
-    let mut compiler = Compiler::new(opts, source_path);
-    compiler.push_new_code_object("<module>".to_owned());
+    let mut compiler = Compiler::new(opts, source_path, "<module>".to_owned());
     f(&mut compiler)?;
     let code = compiler.pop_code_object();
     trace!("Compilation completed: {:?}", code);
@@ -176,9 +193,25 @@ pub fn compile_program_single(
 }
 
 impl Compiler {
-    fn new(opts: CompileOpts, source_path: String) -> Self {
+    fn new(opts: CompileOpts, source_path: String, code_name: String) -> Self {
+        let module_code = CodeInfo {
+            code: CodeObject::new(
+                Default::default(),
+                0,
+                0,
+                0,
+                source_path.clone(),
+                0,
+                code_name,
+            ),
+            name_cache: IndexSet::new(),
+            varname_cache: IndexSet::new(),
+            cellvar_cache: IndexSet::new(),
+            freevar_cache: IndexSet::new(),
+            label_map: Vec::new(),
+        };
         Compiler {
-            code_stack: Vec::new(),
+            code_stack: vec![module_code],
             symbol_table_stack: Vec::new(),
             source_path,
             current_source_location: ast::Location::default(),
@@ -186,6 +219,7 @@ impl Compiler {
             done_with_future_stmts: false,
             ctx: CompileContext {
                 in_loop: false,
+                in_class: false,
                 func: FunctionContext::NoFunction,
             },
             opts,
@@ -204,43 +238,63 @@ impl Compiler {
     }
 
     fn push_output(&mut self, code: CodeObject) {
-        self.code_stack.push(CodeInfo {
+        let table = self
+            .symbol_table_stack
+            .last_mut()
+            .unwrap()
+            .sub_tables
+            .remove(0);
+
+        let cellvar_cache = table
+            .symbols
+            .iter()
+            .filter(|(_, s)| matches!(s.scope, SymbolScope::Cell))
+            .map(|(var, _)| var.clone())
+            .collect();
+        let freevar_cache = table
+            .symbols
+            .iter()
+            // TODO: check if Free or FREE_CLASS symbol
+            .filter(|(_, s)| matches!(s.scope, SymbolScope::Free))
+            .map(|(var, _)| var.clone())
+            .collect();
+
+        self.symbol_table_stack.push(table);
+
+        let info = CodeInfo {
             code,
             name_cache: IndexSet::new(),
+            varname_cache: IndexSet::new(),
+            cellvar_cache,
+            freevar_cache,
             label_map: Vec::new(),
-        });
-    }
-
-    fn push_new_code_object(&mut self, obj_name: String) {
-        let line_number = self.get_source_line_number();
-        self.push_output(CodeObject::new(
-            Default::default(),
-            0,
-            0,
-            0,
-            self.source_path.clone(),
-            line_number,
-            obj_name,
-        ));
+        };
+        self.code_stack.push(info);
     }
 
     fn pop_code_object(&mut self) -> CodeObject {
+        let table = self.symbol_table_stack.pop().unwrap();
+        assert!(table.sub_tables.is_empty());
         self.code_stack.pop().unwrap().finalize_code()
     }
 
     // could take impl Into<Cow<str>>, but everything is borrowed from ast structs; we never
     // actually have a `String` to pass
     fn name(&mut self, name: &str) -> bytecode::NameIdx {
-        let cache = &mut self
-            .code_stack
-            .last_mut()
-            .expect("nothing on stack")
-            .name_cache;
-        if let Some(x) = cache.get_index_of(name) {
-            x
-        } else {
-            cache.insert_full(name.to_owned()).0
-        }
+        self._name_inner(name, |i| &mut i.name_cache)
+    }
+    fn varname(&mut self, name: &str) -> bytecode::NameIdx {
+        self._name_inner(name, |i| &mut i.varname_cache)
+    }
+    fn _name_inner(
+        &mut self,
+        name: &str,
+        cache: impl FnOnce(&mut CodeInfo) -> &mut IndexSet<String>,
+    ) -> bytecode::NameIdx {
+        let cache = cache(self.code_stack.last_mut().expect("nothing on stack"));
+        cache
+            .get_index_of(name)
+            .unwrap_or_else(|| cache.insert_full(name.to_owned()).0)
     }
 
     fn compile_program(
@@ -255,10 +309,7 @@ impl Compiler {
         if let Some(value) = doc {
             self.emit_constant(bytecode::ConstantData::Str { value });
             let doc = self.name("__doc__");
-            self.emit(Instruction::StoreName {
-                idx: doc,
-                scope: bytecode::NameScope::Global,
-            });
+            self.emit(Instruction::StoreGlobal(doc))
         }
         self.compile_statements(statements)?;
 
@@ -331,34 +382,73 @@ impl Compiler {
         Ok(())
     }
 
-    fn scope_for_name(&self, name: &str) -> bytecode::NameScope {
-        let symbol = self.lookup_name(name);
-        match symbol.scope {
-            SymbolScope::Global => bytecode::NameScope::Global,
-            SymbolScope::Nonlocal => bytecode::NameScope::NonLocal,
-            SymbolScope::Unknown => bytecode::NameScope::Free,
-            SymbolScope::Local => {
-                // Only in function block, we use load local
-                // https://github.com/python/cpython/blob/master/Python/compile.c#L3582
-                if self.ctx.in_func() {
-                    bytecode::NameScope::Local
-                } else {
-                    bytecode::NameScope::Free
-                }
-            }
-        }
-    }
-
     fn load_name(&mut self, name: &str) {
-        let scope = self.scope_for_name(name);
-        let idx = self.name(name);
-        self.emit(Instruction::LoadName { idx, scope });
+        self.compile_name(name, NameUsage::Load)
     }
 
     fn store_name(&mut self, name: &str) {
-        let scope = self.scope_for_name(name);
-        let idx = self.name(name);
-        self.emit(Instruction::StoreName { idx, scope });
+        self.compile_name(name, NameUsage::Store)
+    }
+
+    fn compile_name(&mut self, name: &str, usage: NameUsage) {
+        let symbol_table = self.symbol_table_stack.last().unwrap();
+        let symbol = symbol_table.lookup(name).expect(
+            "The symbol must be present in the symbol table, even when it is undefined in python.",
+        );
+        let info = self.code_stack.last_mut().unwrap();
+        let mut cache = &mut info.name_cache;
+        enum NameOpType {
+            Fast,
+            Global,
+            Deref,
+            Local,
+        }
+        let op_typ = match symbol.scope {
+            SymbolScope::Local if self.ctx.in_func() => {
+                cache = &mut info.varname_cache;
+                NameOpType::Fast
+            }
+            SymbolScope::Local => NameOpType::Local,
+            SymbolScope::GlobalImplicit if self.ctx.in_func() => NameOpType::Global,
+            SymbolScope::GlobalImplicit => NameOpType::Local,
+            SymbolScope::GlobalExplicit => NameOpType::Global,
+            SymbolScope::Free => {
+                cache = &mut info.freevar_cache;
+                NameOpType::Deref
+            }
+            SymbolScope::Cell => {
+                cache = &mut info.cellvar_cache;
+                NameOpType::Deref
+            }
+            // TODO: is this right?
+            SymbolScope::Unknown => NameOpType::Global,
+        };
+        let idx = cache
+            .get_index_of(name)
+            .unwrap_or_else(|| cache.insert_full(name.to_owned()).0);
+        let op = match op_typ {
+            NameOpType::Fast => match usage {
+                NameUsage::Load => Instruction::LoadFast,
+                NameUsage::Store => Instruction::StoreFast,
+                NameUsage::Delete => Instruction::DeleteFast,
+            },
+            NameOpType::Global => match usage {
+                NameUsage::Load => Instruction::LoadGlobal,
+                NameUsage::Store => Instruction::StoreGlobal,
+                NameUsage::Delete => Instruction::DeleteGlobal,
+            },
+            NameOpType::Deref => match usage {
+                NameUsage::Load => Instruction::LoadDeref,
+                NameUsage::Store => Instruction::StoreDeref,
+                NameUsage::Delete => Instruction::DeleteDeref,
+            },
+            NameOpType::Local => match usage {
+                NameUsage::Load => Instruction::LoadLocal,
+                NameUsage::Store => Instruction::StoreLocal,
+                NameUsage::Delete => Instruction::DeleteLocal,
+            },
+        };
+        self.emit(op(idx));
     }
 
     fn compile_statement(&mut self, statement: &ast::Statement) -> CompileResult<()> {
@@ -584,10 +674,7 @@ impl Compiler {
                     let end_label = self.new_label();
                     self.compile_jump_if(test, true, end_label)?;
                     let assertion_error = self.name("AssertionError");
-                    self.emit(Instruction::LoadName {
-                        idx: assertion_error,
-                        scope: bytecode::NameScope::Global,
-                    });
+                    self.emit(Instruction::LoadGlobal(assertion_error));
                     match msg {
                         Some(e) => {
                             self.compile_expression(e)?;
@@ -683,8 +770,7 @@ impl Compiler {
     fn compile_delete(&mut self, expression: &ast::Expression) -> CompileResult<()> {
         match &expression.node {
             ast::ExpressionType::Identifier { name } => {
-                let idx = self.name(name);
-                self.emit(Instruction::DeleteName { idx });
+                self.compile_name(name, NameUsage::Delete);
             }
             ast::ExpressionType::Attribute { value, name } => {
                 self.compile_expression(value)?;
@@ -738,7 +824,7 @@ impl Compiler {
             });
         }
 
-        let mut flags = bytecode::CodeFlags::default();
+        let mut flags = bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED;
         if have_defaults {
             flags |= bytecode::CodeFlags::HAS_DEFAULTS;
         }
@@ -775,8 +861,6 @@ impl Compiler {
         compile_varargs(&args.vararg, bytecode::CodeFlags::HAS_VARARGS);
         compile_varargs(&args.kwarg, bytecode::CodeFlags::HAS_VARKEYWORDS);
 
-        self.enter_scope();
-
         Ok(())
     }
 
@@ -923,6 +1007,7 @@ impl Compiler {
 
         self.ctx = CompileContext {
             in_loop: false,
+            in_class: prev_ctx.in_class,
             func: if is_async {
                 FunctionContext::AsyncFunction
             } else {
@@ -954,7 +1039,6 @@ impl Compiler {
         }
 
         let mut code = self.pop_code_object();
-        self.leave_scope();
 
         // Prepare type annotations:
         let mut num_annotations = 0;
@@ -1006,6 +1090,27 @@ impl Compiler {
             code.flags |= bytecode::CodeFlags::IS_COROUTINE;
         }
 
+        if !code.freevars.is_empty() {
+            for var in &code.freevars {
+                let symbol = self.symbol_table_stack.last().unwrap().lookup(var).unwrap();
+                let parent_code = self.code_stack.last().unwrap();
+                let vars = match symbol.scope {
+                    SymbolScope::Free => &parent_code.freevar_cache,
+                    SymbolScope::Cell => &parent_code.cellvar_cache,
+                    _ => unreachable!(),
+                };
+                let mut idx = vars.get_index_of(var).unwrap();
+                if let SymbolScope::Free = symbol.scope {
+                    idx += parent_code.cellvar_cache.len();
+                }
+                self.emit(Instruction::LoadClosure(idx))
+            }
+            self.emit(Instruction::BuildTuple {
+                size: code.freevars.len(),
+                unpack: false,
+            })
+        }
+
         self.emit_constant(bytecode::ConstantData::Code {
             code: Box::new(code),
         });
@@ -1098,6 +1203,7 @@ impl Compiler {
         let prev_ctx = self.ctx;
         self.ctx = CompileContext {
             func: FunctionContext::NoFunction,
+            in_class: true,
             in_loop: false,
         };
 
@@ -1109,7 +1215,7 @@ impl Compiler {
         self.emit(Instruction::LoadBuildClass);
         let line_number = self.get_source_line_number();
         self.push_output(CodeObject::new(
-            Default::default(),
+            bytecode::CodeFlags::empty(),
             0,
             0,
             0,
@@ -1117,34 +1223,21 @@ impl Compiler {
             line_number,
             name.to_owned(),
         ));
-        self.enter_scope();
 
         let (new_body, doc_str) = get_doc(body);
 
         let dunder_name = self.name("__name__");
-        self.emit(Instruction::LoadName {
-            idx: dunder_name,
-            scope: bytecode::NameScope::Global,
-        });
+        self.emit(Instruction::LoadGlobal(dunder_name));
         let dunder_module = self.name("__module__");
-        self.emit(Instruction::StoreName {
-            idx: dunder_module,
-            scope: bytecode::NameScope::Free,
-        });
+        self.emit(Instruction::StoreLocal(dunder_module));
         self.emit_constant(bytecode::ConstantData::Str {
             value: qualified_name.clone(),
         });
         let qualname = self.name("__qualname__");
-        self.emit(Instruction::StoreName {
-            idx: qualname,
-            scope: bytecode::NameScope::Free,
-        });
+        self.emit(Instruction::StoreLocal(qualname));
         self.load_docstring(doc_str);
         let doc = self.name("__doc__");
-        self.emit(Instruction::StoreName {
-            idx: doc,
-            scope: bytecode::NameScope::Free,
-        });
+        self.emit(Instruction::StoreLocal(doc));
         // setup annotations
         if self.find_ann(body) {
             self.emit(Instruction::SetupAnnotation);
@@ -1153,9 +1246,7 @@ impl Compiler {
         self.emit_constant(bytecode::ConstantData::None);
         self.emit(Instruction::ReturnValue);
 
-        let mut code = self.pop_code_object();
-        code.flags.remove(bytecode::CodeFlags::NEW_LOCALS);
-        self.leave_scope();
+        let code = self.pop_code_object();
 
         self.emit_constant(bytecode::ConstantData::Code {
             code: Box::new(code),
@@ -1295,10 +1386,7 @@ impl Compiler {
             self.set_label(check_asynciter_label);
             self.emit(Instruction::Duplicate);
             let stopasynciter = self.name("StopAsyncIteration");
-            self.emit(Instruction::LoadName {
-                idx: stopasynciter,
-                scope: bytecode::NameScope::Global,
-            });
+            self.emit(Instruction::LoadGlobal(stopasynciter));
             self.emit(Instruction::CompareOperation {
                 op: bytecode::ComparisonOperator::ExceptionMatch,
             });
@@ -1436,15 +1524,14 @@ impl Compiler {
 
         if let ast::ExpressionType::Identifier { name } = &target.node {
             // Store as dict entry in __annotations__ dict:
-            let annotations = self.name("__annotations__");
-            self.emit(Instruction::LoadName {
-                idx: annotations,
-                scope: bytecode::NameScope::Local,
-            });
-            self.emit_constant(bytecode::ConstantData::Str {
-                value: name.to_owned(),
-            });
-            self.emit(Instruction::StoreSubscript);
+            if !self.ctx.in_func() {
+                let annotations = self.name("__annotations__");
+                self.emit(Instruction::LoadLocal(annotations));
+                self.emit_constant(bytecode::ConstantData::Str {
+                    value: name.to_owned(),
+                });
+                self.emit(Instruction::StoreSubscript);
+            }
         } else {
             // Drop annotation if not assigned to simple identifier.
             self.emit(Instruction::Pop);
@@ -1846,6 +1933,7 @@ impl Compiler {
                 let prev_ctx = self.ctx;
                 self.ctx = CompileContext {
                     in_loop: false,
+                    in_class: prev_ctx.in_class,
                     func: FunctionContext::Function,
                 };
 
@@ -1854,7 +1942,6 @@ impl Compiler {
                 self.compile_expression(body)?;
                 self.emit(Instruction::ReturnValue);
                 let code = self.pop_code_object();
-                self.leave_scope();
                 self.emit_constant(bytecode::ConstantData::Code {
                     code: Box::new(code),
                 });
@@ -2044,8 +2131,7 @@ impl Compiler {
             line_number,
             name.clone(),
         ));
-        let arg0 = self.name(".0");
-        self.enter_scope();
+        let arg0 = self.varname(".0");
 
         // Create empty object of proper type:
         match kind {
@@ -2079,10 +2165,7 @@ impl Compiler {
 
             if loop_labels.is_empty() {
                 // Load iterator onto stack (passed as first argument):
-                self.emit(Instruction::LoadName {
-                    idx: arg0,
-                    scope: bytecode::NameScope::Local,
-                });
+                self.emit(Instruction::LoadFast(arg0));
             } else {
                 // Evaluate iterated item:
                 self.compile_expression(&generator.iter)?;
@@ -2169,9 +2252,6 @@ impl Compiler {
         // Fetch code for listcomp function:
         let code = self.pop_code_object();
 
-        // Pop scope
-        self.leave_scope();
-
         // List comprehension code:
         self.emit_constant(bytecode::ConstantData::Code {
             code: Box::new(code),
@@ -2255,33 +2335,6 @@ impl Compiler {
         Ok(())
     }
 
-    // Scope helpers:
-    fn enter_scope(&mut self) {
-        // println!("Enter scope {:?}", self.symbol_table_stack);
-        // Enter first subscope!
-        let table = self
-            .symbol_table_stack
-            .last_mut()
-            .unwrap()
-            .sub_tables
-            .remove(0);
-        self.symbol_table_stack.push(table);
-    }
-
-    fn leave_scope(&mut self) {
-        // println!("Leave scope {:?}", self.symbol_table_stack);
-        let table = self.symbol_table_stack.pop().unwrap();
-        assert!(table.sub_tables.is_empty());
-    }
-
-    fn lookup_name(&self, name: &str) -> &Symbol {
-        // println!("Looking up {:?}", name);
-        let symbol_table = self.symbol_table_stack.last().unwrap();
-        symbol_table.lookup(name).expect(
-            "The symbol must be present in the symbol table, even when it is undefined in python.",
-        )
-    }
-
     // Low level helper functions:
     fn emit(&mut self, instruction: Instruction) {
         let location = compile_location(&self.current_source_location);
@@ -2402,9 +2455,11 @@ mod tests {
     use rustpython_parser::parser;
 
     fn compile_exec(source: &str) -> CodeObject {
-        let mut compiler: Compiler =
-            Compiler::new(CompileOpts::default(), "source_path".to_owned());
-        compiler.push_new_code_object("<module>".to_owned());
+        let mut compiler: Compiler = Compiler::new(
+            CompileOpts::default(),
+            "source_path".to_owned(),
+            "<module>".to_owned(),
+        );
         let ast = parser::parse_program(source).unwrap();
         let symbol_scope = make_symbol_table(&ast).unwrap();
         compiler.compile_program(&ast, symbol_scope).unwrap();
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 276e55c4..02f15ae7 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -64,7 +64,7 @@ impl SymbolTable {
     }
 }
 
-#[derive(Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq)]
 pub enum SymbolTableType {
     Module,
     Class,
@@ -85,12 +85,14 @@ impl fmt::Display for SymbolTableType {
 
 /// Indicator for a single symbol what the scope of this symbol is.
 /// The scope can be unknown, which is unfortunate, but not impossible.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Copy)]
 pub enum SymbolScope {
-    Global,
-    Nonlocal,
-    Local,
     Unknown,
+    Local,
+    GlobalExplicit,
+    GlobalImplicit,
+    Free,
+    Cell,
 }
 
 /// A single symbol in a table. Has various properties such as the scope
@@ -104,7 +106,6 @@ pub struct Symbol {
     pub is_referenced: bool,
     pub is_assigned: bool,
     pub is_parameter: bool,
-    pub is_free: bool,
     pub is_annotated: bool,
     pub is_imported: bool,
 
@@ -126,7 +127,6 @@ impl Symbol {
             is_referenced: false,
             is_assigned: false,
             is_parameter: false,
-            is_free: false,
             is_annotated: false,
             is_imported: false,
             is_assign_namedexpr_in_comprehension: false,
@@ -135,7 +135,10 @@ impl Symbol {
     }
 
     pub fn is_global(&self) -> bool {
-        matches!(self.scope, SymbolScope::Global)
+        matches!(
+            self.scope,
+            SymbolScope::GlobalExplicit | SymbolScope::GlobalImplicit
+        )
     }
 
     pub fn is_local(&self) -> bool {
@@ -228,7 +231,7 @@ impl<'a> SymbolTableAnalyzer<'a> {
             self.analyze_symbol_comprehension(symbol, 0)?
         } else {
             match symbol.scope {
-                SymbolScope::Nonlocal => {
+                SymbolScope::Free => {
                     let scope_depth = self.tables.len();
                     if scope_depth > 0 {
                         // check if the name is already defined in any outer scope
@@ -251,10 +254,10 @@ impl<'a> SymbolTableAnalyzer<'a> {
                         });
                     }
                 }
-                SymbolScope::Global => {
+                SymbolScope::GlobalExplicit | SymbolScope::GlobalImplicit => {
                     // TODO: add more checks for globals?
                 }
-                SymbolScope::Local => {
+                SymbolScope::Local | SymbolScope::Cell => {
                     // all is well
                 }
                 SymbolScope::Unknown => {
@@ -270,24 +273,28 @@ impl<'a> SymbolTableAnalyzer<'a> {
         // Interesting stuff about the __class__ variable:
         // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
         symbol.name == "__class__"
-            || self.tables.iter().skip(1).any(|(symbols, typ)| {
-                *typ != SymbolTableType::Class && symbols.contains_key(&symbol.name)
+            || self.tables.iter().skip(1).rev().any(|(symbols, typ)| {
+                *typ != SymbolTableType::Class
+                    && symbols
+                        .get(&symbol.name)
+                        .map_or(false, |sym| sym.is_local() && sym.is_assigned)
             })
     }
 
     fn analyze_unknown_symbol(&self, symbol: &mut Symbol) {
-        if symbol.is_assigned || symbol.is_parameter {
-            symbol.scope = SymbolScope::Local;
+        let scope = if symbol.is_assigned || symbol.is_parameter {
+            SymbolScope::Local
         } else if self.found_in_outer_scope(symbol) {
             // Symbol is in some outer scope.
-            symbol.is_free = true;
+            SymbolScope::Free
         } else if self.tables.is_empty() {
             // Don't make assumptions when we don't know.
-            symbol.scope = SymbolScope::Unknown;
+            SymbolScope::Unknown
         } else {
-            // If there are scopes above we can assume global.
-            symbol.scope = SymbolScope::Global;
-        }
+            // If there are scopes above we assume global.
+            SymbolScope::GlobalImplicit
+        };
+        symbol.scope = scope;
     }
 
     // Implements the symbol analysis and scope extension for names
@@ -302,7 +309,7 @@ impl<'a> SymbolTableAnalyzer<'a> {
         // when this is called, we expect to be in the direct parent scope of the scope that contains 'symbol'
         let offs = self.tables.len() - 1 - parent_offset;
         let last = self.tables.get_mut(offs).unwrap();
-        let symbols = &mut last.0;
+        let symbols = &mut *last.0;
         let table_type = last.1;
 
         // it is not allowed to use an iterator variable as assignee in a named expression
@@ -319,7 +326,7 @@ impl<'a> SymbolTableAnalyzer<'a> {
 
         match table_type {
             SymbolTableType::Module => {
-                symbol.scope = SymbolScope::Global;
+                symbol.scope = SymbolScope::GlobalImplicit;
             }
             SymbolTableType::Class => {
                 // named expressions are forbidden in comprehensions on class scope
@@ -327,22 +334,18 @@ impl<'a> SymbolTableAnalyzer<'a> {
                     error: "assignment expression within a comprehension cannot be used in a class body".to_string(),
                     // TODO: accurate location info, somehow
                     location: Location::default(),
-                } );
+                });
             }
             SymbolTableType::Function => {
                 if let Some(parent_symbol) = symbols.get_mut(&symbol.name) {
                     if let SymbolScope::Unknown = parent_symbol.scope {
-                        parent_symbol.is_assigned = true; // this information is new, as the asignment is done in inner scope
-                                                          //self.analyze_unknown_symbol(symbol); // not needed, symbol is analyzed anyhow when its scope is analyzed
+                        // this information is new, as the asignment is done in inner scope
+                        parent_symbol.is_assigned = true;
+                        //self.analyze_unknown_symbol(symbol); // not needed, symbol is analyzed anyhow when its scope is analyzed
                     }
 
-                    match symbol.scope {
-                        SymbolScope::Global => {
-                            symbol.scope = SymbolScope::Global;
-                        }
-                        _ => {
-                            symbol.scope = SymbolScope::Nonlocal;
-                        }
+                    if !symbol.is_global() {
+                        symbol.scope = SymbolScope::Free;
                     }
                 } else {
                     let mut cloned_sym = symbol.clone();
@@ -372,7 +375,7 @@ impl<'a> SymbolTableAnalyzer<'a> {
                         // ouside, too, and set it therefore to non-local scope. I.e., we expect to
                         // find a definition on a higher level
                         let mut cloned_sym = symbol.clone();
-                        cloned_sym.scope = SymbolScope::Nonlocal;
+                        cloned_sym.scope = SymbolScope::Free;
                         last.0.insert(cloned_sym.name.to_owned(), cloned_sym);
                     }
                 }
@@ -408,7 +411,7 @@ struct SymbolTableBuilder {
 /// was used.
 /// In cpython this is stored in the AST, but I think this
 /// is not logical, since it is not context free.
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, PartialEq)]
 enum ExpressionContext {
     Load,
     Store,
@@ -825,6 +828,12 @@ impl SymbolTableBuilder {
                         self.register_name(name, SymbolUsage::Iter, location)?;
                     }
                 }
+                if context == ExpressionContext::Load
+                    && self.tables.last().unwrap().typ == SymbolTableType::Function
+                    && name == "super"
+                {
+                    self.register_name("__class__", SymbolUsage::Used, location)?;
+                }
             }
             Lambda { args, body } => {
                 self.enter_function("lambda", args, expression.location.row())?;
@@ -949,7 +958,7 @@ impl SymbolTableBuilder {
             // Role already set..
             match role {
                 SymbolUsage::Global => {
-                    if let SymbolScope::Global = symbol.scope {
+                    if symbol.is_global() {
                         // Ok
                     } else {
                         return Err(SymbolTableError {
@@ -1014,7 +1023,7 @@ impl SymbolTableBuilder {
         let symbol = table.symbols.get_mut(name).unwrap();
         match role {
             SymbolUsage::Nonlocal => {
-                symbol.scope = SymbolScope::Nonlocal;
+                symbol.scope = SymbolScope::Free;
             }
             SymbolUsage::Imported => {
                 symbol.is_assigned = true;
@@ -1040,8 +1049,8 @@ impl SymbolTableBuilder {
             }
             SymbolUsage::Global => {
                 if let SymbolScope::Unknown = symbol.scope {
-                    symbol.scope = SymbolScope::Global;
-                } else if let SymbolScope::Global = symbol.scope {
+                    symbol.scope = SymbolScope::GlobalImplicit;
+                } else if symbol.is_global() {
                     // Global scope can be set to global
                 } else {
                     return Err(SymbolTableError {

From e18275566aabf50f720492870063a70b80ba3350 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sun, 29 Nov 2020 20:16:01 -0600
Subject: [PATCH 169/459] Fast locals part 2

---
 src/compile.rs     | 146 ++++++++++++++++++++++++++++-----------
 src/symboltable.rs | 168 ++++++++++++++++++++++++++++++++++-----------
 2 files changed, 236 insertions(+), 78 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 3137b47b..2917ccd4 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -40,6 +40,31 @@ impl CodeInfo {
         code.cellvars.extend(cellvar_cache);
         code.freevars.extend(freevar_cache);
 
+        if !code.cellvars.is_empty() {
+            let total_args = code.arg_count
+                + code.kwonlyarg_count
+                + code.flags.contains(bytecode::CodeFlags::HAS_VARARGS) as usize
+                + code.flags.contains(bytecode::CodeFlags::HAS_VARKEYWORDS) as usize;
+            let all_args = &code.varnames[..total_args];
+            let mut found_cellarg = false;
+            let cell2arg = code
+                .cellvars
+                .iter()
+                .map(|var| {
+                    for (i, arg) in all_args.iter().enumerate() {
+                        if var == arg {
+                            found_cellarg = true;
+                            return i as isize;
+                        }
+                    }
+                    -1
+                })
+                .collect::<Box<[_]>>();
+            if found_cellarg {
+                code.cell2arg = Some(cell2arg);
+            }
+        }
+
         for instruction in &mut code.instructions {
             use Instruction::*;
             // this is a little bit hacky, as until now the data stored inside Labels in
@@ -64,7 +89,7 @@ impl CodeInfo {
                 }
 
                 #[rustfmt::skip]
-                Import { .. } | ImportStar | ImportFrom { .. } | LoadFast(_) | LoadLocal(_)
+                Import { .. } | ImportStar | ImportFrom { .. } | LoadFast(_) | LoadNameAny(_)
                 | LoadGlobal(_) | LoadDeref(_) | LoadClassDeref(_) | StoreFast(_) | StoreLocal(_)
                 | StoreGlobal(_) | StoreDeref(_) | DeleteFast(_) | DeleteLocal(_) | DeleteGlobal(_)
                 | DeleteDeref(_) | LoadClosure(_) | Subscript | StoreSubscript | DeleteSubscript
@@ -114,14 +139,14 @@ impl Default for CompileOpts {
     }
 }
 
-#[derive(Clone, Copy)]
+#[derive(Debug, Clone, Copy)]
 struct CompileContext {
     in_loop: bool,
     in_class: bool,
     func: FunctionContext,
 }
 
-#[derive(Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq)]
 enum FunctionContext {
     NoFunction,
     Function,
@@ -130,7 +155,7 @@ enum FunctionContext {
 
 impl CompileContext {
     fn in_func(self) -> bool {
-        !matches!(self.func, FunctionContext::NoFunction)
+        self.func != FunctionContext::NoFunction
     }
 }
 
@@ -408,9 +433,8 @@ impl Compiler {
                 cache = &mut info.varname_cache;
                 NameOpType::Fast
             }
-            SymbolScope::Local => NameOpType::Local,
             SymbolScope::GlobalImplicit if self.ctx.in_func() => NameOpType::Global,
-            SymbolScope::GlobalImplicit => NameOpType::Local,
+            SymbolScope::Local | SymbolScope::GlobalImplicit => NameOpType::Local,
             SymbolScope::GlobalExplicit => NameOpType::Global,
             SymbolScope::Free => {
                 cache = &mut info.freevar_cache;
@@ -423,9 +447,12 @@ impl Compiler {
             // TODO: is this right?
             SymbolScope::Unknown => NameOpType::Global,
         };
-        let idx = cache
+        let mut idx = cache
             .get_index_of(name)
             .unwrap_or_else(|| cache.insert_full(name.to_owned()).0);
+        if let SymbolScope::Free = symbol.scope {
+            idx += info.cellvar_cache.len();
+        }
         let op = match op_typ {
             NameOpType::Fast => match usage {
                 NameUsage::Load => Instruction::LoadFast,
@@ -438,12 +465,15 @@ impl Compiler {
                 NameUsage::Delete => Instruction::DeleteGlobal,
             },
             NameOpType::Deref => match usage {
+                NameUsage::Load if !self.ctx.in_func() && self.ctx.in_class => {
+                    Instruction::LoadClassDeref
+                }
                 NameUsage::Load => Instruction::LoadDeref,
                 NameUsage::Store => Instruction::StoreDeref,
                 NameUsage::Delete => Instruction::DeleteDeref,
             },
             NameOpType::Local => match usage {
-                NameUsage::Load => Instruction::LoadLocal,
+                NameUsage::Load => Instruction::LoadNameAny,
                 NameUsage::Store => Instruction::StoreLocal,
                 NameUsage::Delete => Instruction::DeleteLocal,
             },
@@ -844,17 +874,17 @@ impl Compiler {
         ));
 
         for name in &args.args {
-            self.name(&name.arg);
+            self.varname(&name.arg);
         }
         for name in &args.kwonlyargs {
-            self.name(&name.arg);
+            self.varname(&name.arg);
         }
 
         let mut compile_varargs = |va: &ast::Varargs, flag| match va {
             ast::Varargs::None | ast::Varargs::Unnamed => {}
             ast::Varargs::Named(name) => {
                 self.current_code().flags |= flag;
-                self.name(&name.arg);
+                self.varname(&name.arg);
             }
         };
 
@@ -1002,6 +1032,10 @@ impl Compiler {
         is_async: bool,
     ) -> CompileResult<()> {
         // Create bytecode for this function:
+
+        self.prepare_decorators(decorator_list)?;
+        self.enter_function(name, args)?;
+
         // remember to restore self.ctx.in_loop to the original after the function is compiled
         let prev_ctx = self.ctx;
 
@@ -1019,10 +1053,6 @@ impl Compiler {
         let old_qualified_path = self.current_qualified_path.take();
         self.current_qualified_path = Some(self.create_qualified_name(name, ".<locals>"));
 
-        self.prepare_decorators(decorator_list)?;
-
-        self.enter_function(name, args)?;
-
         let (body, doc_str) = get_doc(body);
 
         self.compile_statements(body)?;
@@ -1090,26 +1120,7 @@ impl Compiler {
             code.flags |= bytecode::CodeFlags::IS_COROUTINE;
         }
 
-        if !code.freevars.is_empty() {
-            for var in &code.freevars {
-                let symbol = self.symbol_table_stack.last().unwrap().lookup(var).unwrap();
-                let parent_code = self.code_stack.last().unwrap();
-                let vars = match symbol.scope {
-                    SymbolScope::Free => &parent_code.freevar_cache,
-                    SymbolScope::Cell => &parent_code.cellvar_cache,
-                    _ => unreachable!(),
-                };
-                let mut idx = vars.get_index_of(var).unwrap();
-                if let SymbolScope::Free = symbol.scope {
-                    idx += parent_code.cellvar_cache.len();
-                }
-                self.emit(Instruction::LoadClosure(idx))
-            }
-            self.emit(Instruction::BuildTuple {
-                size: code.freevars.len(),
-                unpack: false,
-            })
-        }
+        self.build_closure(&code);
 
         self.emit_constant(bytecode::ConstantData::Code {
             code: Box::new(code),
@@ -1126,15 +1137,40 @@ impl Compiler {
         self.emit(Instruction::Rotate { amount: 2 });
         let doc = self.name("__doc__");
         self.emit(Instruction::StoreAttr { idx: doc });
+
+        self.current_qualified_path = old_qualified_path;
+        self.ctx = prev_ctx;
+
         self.apply_decorators(decorator_list);
 
         self.store_name(name);
 
-        self.current_qualified_path = old_qualified_path;
-        self.ctx = prev_ctx;
         Ok(())
     }
 
+    fn build_closure(&mut self, code: &CodeObject) {
+        if !code.freevars.is_empty() {
+            for var in &code.freevars {
+                let symbol = self.symbol_table_stack.last().unwrap().lookup(var).unwrap();
+                let parent_code = self.code_stack.last().unwrap();
+                let vars = match symbol.scope {
+                    SymbolScope::Free => &parent_code.freevar_cache,
+                    SymbolScope::Cell => &parent_code.cellvar_cache,
+                    _ => unreachable!(),
+                };
+                let mut idx = vars.get_index_of(var).unwrap();
+                if let SymbolScope::Free = symbol.scope {
+                    idx += parent_code.cellvar_cache.len();
+                }
+                self.emit(Instruction::LoadClosure(idx))
+            }
+            self.emit(Instruction::BuildTuple {
+                size: code.freevars.len(),
+                unpack: false,
+            })
+        }
+    }
+
     fn find_ann(&self, body: &[ast::Statement]) -> bool {
         use ast::StatementType::*;
         let option_stmt_to_bool = |suit: &Option<ast::Suite>| -> bool {
@@ -1243,11 +1279,30 @@ impl Compiler {
             self.emit(Instruction::SetupAnnotation);
         }
         self.compile_statements(new_body)?;
-        self.emit_constant(bytecode::ConstantData::None);
+
+        let classcell_idx = self
+            .code_stack
+            .last_mut()
+            .unwrap()
+            .cellvar_cache
+            .iter()
+            .position(|var| *var == "__class__");
+
+        if let Some(classcell_idx) = classcell_idx {
+            self.emit(Instruction::LoadClosure(classcell_idx));
+            self.emit(Instruction::Duplicate);
+            let classcell = self.name("__classcell__");
+            self.emit(Instruction::StoreLocal(classcell));
+        } else {
+            self.emit_constant(bytecode::ConstantData::None);
+        }
+
         self.emit(Instruction::ReturnValue);
 
         let code = self.pop_code_object();
 
+        self.build_closure(&code);
+
         self.emit_constant(bytecode::ConstantData::Code {
             code: Box::new(code),
         });
@@ -1526,7 +1581,7 @@ impl Compiler {
             // Store as dict entry in __annotations__ dict:
             if !self.ctx.in_func() {
                 let annotations = self.name("__annotations__");
-                self.emit(Instruction::LoadLocal(annotations));
+                self.emit(Instruction::LoadNameAny(annotations));
                 self.emit_constant(bytecode::ConstantData::Str {
                     value: name.to_owned(),
                 });
@@ -1942,6 +1997,7 @@ impl Compiler {
                 self.compile_expression(body)?;
                 self.emit(Instruction::ReturnValue);
                 let code = self.pop_code_object();
+                self.build_closure(&code);
                 self.emit_constant(bytecode::ConstantData::Code {
                     code: Box::new(code),
                 });
@@ -2109,6 +2165,14 @@ impl Compiler {
         kind: &ast::ComprehensionKind,
         generators: &[ast::Comprehension],
     ) -> CompileResult<()> {
+        let prev_ctx = self.ctx;
+
+        self.ctx = CompileContext {
+            in_loop: false,
+            in_class: prev_ctx.in_class,
+            func: FunctionContext::Function,
+        };
+
         // We must have at least one generator:
         assert!(!generators.is_empty());
 
@@ -2252,6 +2316,10 @@ impl Compiler {
         // Fetch code for listcomp function:
         let code = self.pop_code_object();
 
+        self.ctx = prev_ctx;
+
+        self.build_closure(&code);
+
         // List comprehension code:
         self.emit_constant(bytecode::ConstantData::Code {
             code: Box::new(code),
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 02f15ae7..ce706208 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -144,6 +144,10 @@ impl Symbol {
     pub fn is_local(&self) -> bool {
         matches!(self.scope, SymbolScope::Local)
     }
+
+    pub fn is_bound(&self) -> bool {
+        self.is_assigned || self.is_parameter || self.is_imported || self.is_iter
+    }
 }
 
 #[derive(Debug)]
@@ -189,29 +193,89 @@ fn analyze_symbol_table(symbol_table: &mut SymbolTable) -> SymbolTableResult {
     analyzer.analyze_symbol_table(symbol_table)
 }
 
+type SymbolMap = IndexMap<String, Symbol>;
+
+mod stack {
+    use std::panic;
+    use std::ptr::NonNull;
+    pub struct StackStack<T> {
+        v: Vec<NonNull<T>>,
+    }
+    impl<T> Default for StackStack<T> {
+        fn default() -> Self {
+            Self { v: Vec::new() }
+        }
+    }
+    impl<T> StackStack<T> {
+        pub fn append<F, R>(&mut self, x: &mut T, f: F) -> R
+        where
+            F: FnOnce(&mut Self) -> R,
+        {
+            self.v.push(x.into());
+            let res = panic::catch_unwind(panic::AssertUnwindSafe(|| f(self)));
+            self.v.pop();
+            res.unwrap_or_else(|x| panic::resume_unwind(x))
+        }
+
+        pub fn iter(&self) -> impl Iterator<Item = &T> + DoubleEndedIterator + '_ {
+            self.as_ref().iter().copied()
+        }
+        pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut T> + DoubleEndedIterator + '_ {
+            self.as_mut().iter_mut().map(|x| &mut **x)
+        }
+        // pub fn top(&self) -> Option<&T> {
+        //     self.as_ref().last().copied()
+        // }
+        // pub fn top_mut(&mut self) -> Option<&mut T> {
+        //     self.as_mut().last_mut().map(|x| &mut **x)
+        // }
+        pub fn len(&self) -> usize {
+            self.v.len()
+        }
+        pub fn is_empty(&self) -> bool {
+            self.len() == 0
+        }
+
+        pub fn as_ref(&self) -> &[&T] {
+            unsafe { &*(self.v.as_slice() as *const [NonNull<T>] as *const [&T]) }
+        }
+
+        pub fn as_mut(&mut self) -> &mut [&mut T] {
+            unsafe { &mut *(self.v.as_mut_slice() as *mut [NonNull<T>] as *mut [&mut T]) }
+        }
+    }
+}
+use stack::StackStack;
+
 /// Symbol table analysis. Can be used to analyze a fully
 /// build symbol table structure. It will mark variables
 /// as local variables for example.
 #[derive(Default)]
-struct SymbolTableAnalyzer<'a> {
-    tables: Vec<(&'a mut IndexMap<String, Symbol>, SymbolTableType)>,
+#[repr(transparent)]
+struct SymbolTableAnalyzer {
+    tables: StackStack<(SymbolMap, SymbolTableType)>,
 }
 
-impl<'a> SymbolTableAnalyzer<'a> {
-    fn analyze_symbol_table(&mut self, symbol_table: &'a mut SymbolTable) -> SymbolTableResult {
-        let symbols = &mut symbol_table.symbols;
-        let sub_tables = &mut symbol_table.sub_tables;
+impl SymbolTableAnalyzer {
+    fn analyze_symbol_table(&mut self, symbol_table: &mut SymbolTable) -> SymbolTableResult {
+        let symbols = std::mem::take(&mut symbol_table.symbols);
+        let sub_tables = &mut *symbol_table.sub_tables;
+
+        let mut info = (symbols, symbol_table.typ);
+        self.tables.append(&mut info, |list| {
+            let inner_scope = unsafe { &mut *(list as *mut _ as *mut SymbolTableAnalyzer) };
+            // Analyze sub scopes:
+            for sub_table in sub_tables.iter_mut() {
+                inner_scope.analyze_symbol_table(sub_table)?;
+            }
+            Ok(())
+        })?;
 
-        self.tables.push((symbols, symbol_table.typ));
-        // Analyze sub scopes:
-        for sub_table in sub_tables {
-            self.analyze_symbol_table(sub_table)?;
-        }
-        let (symbols, st_typ) = self.tables.pop().unwrap();
+        symbol_table.symbols = info.0;
 
         // Analyze symbols:
-        for symbol in symbols.values_mut() {
-            self.analyze_symbol(symbol, st_typ)?;
+        for symbol in symbol_table.symbols.values_mut() {
+            self.analyze_symbol(symbol, symbol_table.typ, sub_tables)?;
         }
         Ok(())
     }
@@ -220,6 +284,7 @@ impl<'a> SymbolTableAnalyzer<'a> {
         &mut self,
         symbol: &mut Symbol,
         curr_st_typ: SymbolTableType,
+        sub_tables: &mut [SymbolTable],
     ) -> SymbolTableResult {
         if symbol.is_assign_namedexpr_in_comprehension
             && curr_st_typ == SymbolTableType::Comprehension
@@ -232,11 +297,11 @@ impl<'a> SymbolTableAnalyzer<'a> {
         } else {
             match symbol.scope {
                 SymbolScope::Free => {
-                    let scope_depth = self.tables.len();
-                    if scope_depth > 0 {
+                    if !self.tables.as_ref().is_empty() {
+                        let scope_depth = self.tables.as_ref().iter().count();
                         // check if the name is already defined in any outer scope
                         // therefore
-                        if scope_depth < 2 || !self.found_in_outer_scope(symbol) {
+                        if scope_depth < 2 || !self.found_in_outer_scope(&symbol.name) {
                             return Err(SymbolTableError {
                                 error: format!("no binding for nonlocal '{}' found", symbol.name),
                                 // TODO: accurate location info, somehow
@@ -262,29 +327,56 @@ impl<'a> SymbolTableAnalyzer<'a> {
                 }
                 SymbolScope::Unknown => {
                     // Try hard to figure out what the scope of this symbol is.
-                    self.analyze_unknown_symbol(symbol);
+                    self.analyze_unknown_symbol(sub_tables, symbol);
                 }
             }
         }
         Ok(())
     }
 
-    fn found_in_outer_scope(&self, symbol: &Symbol) -> bool {
+    fn found_in_outer_scope(&mut self, name: &str) -> bool {
         // Interesting stuff about the __class__ variable:
         // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
-        symbol.name == "__class__"
-            || self.tables.iter().skip(1).rev().any(|(symbols, typ)| {
-                *typ != SymbolTableType::Class
-                    && symbols
-                        .get(&symbol.name)
-                        .map_or(false, |sym| sym.is_local() && sym.is_assigned)
-            })
+        if name == "__class__" {
+            return true;
+        }
+        let decl_depth = self.tables.iter().rev().position(|(symbols, typ)| {
+            !matches!(typ, SymbolTableType::Class | SymbolTableType::Module)
+                && symbols.get(name).map_or(false, |sym| sym.is_bound())
+        });
+
+        if let Some(decl_depth) = decl_depth {
+            // decl_depth is the number of tables between the current one and
+            // the one that declared the cell var
+            for (table, _) in self.tables.iter_mut().rev().take(decl_depth) {
+                if !table.contains_key(name) {
+                    let mut symbol = Symbol::new(name);
+                    symbol.scope = SymbolScope::Free;
+                    symbol.is_referenced = true;
+                    table.insert(name.to_owned(), symbol);
+                }
+            }
+        }
+
+        decl_depth.is_some()
+    }
+
+    fn found_in_inner_scope(sub_tables: &mut [SymbolTable], name: &str) -> bool {
+        sub_tables.iter().any(|x| {
+            x.symbols
+                .get(name)
+                .map_or(false, |sym| matches!(sym.scope, SymbolScope::Free))
+        })
     }
 
-    fn analyze_unknown_symbol(&self, symbol: &mut Symbol) {
-        let scope = if symbol.is_assigned || symbol.is_parameter {
-            SymbolScope::Local
-        } else if self.found_in_outer_scope(symbol) {
+    fn analyze_unknown_symbol(&mut self, sub_tables: &mut [SymbolTable], symbol: &mut Symbol) {
+        let scope = if symbol.is_bound() {
+            if Self::found_in_inner_scope(sub_tables, &symbol.name) {
+                SymbolScope::Cell
+            } else {
+                SymbolScope::Local
+            }
+        } else if self.found_in_outer_scope(&symbol.name) {
             // Symbol is in some outer scope.
             SymbolScope::Free
         } else if self.tables.is_empty() {
@@ -305,11 +397,9 @@ impl<'a> SymbolTableAnalyzer<'a> {
         symbol: &mut Symbol,
         parent_offset: usize,
     ) -> SymbolTableResult {
-        // TODO: quite C-ish way to implement the iteration
         // when this is called, we expect to be in the direct parent scope of the scope that contains 'symbol'
-        let offs = self.tables.len() - 1 - parent_offset;
-        let last = self.tables.get_mut(offs).unwrap();
-        let symbols = &mut *last.0;
+        let last = self.tables.iter_mut().rev().nth(parent_offset).unwrap();
+        let symbols = &mut last.0;
         let table_type = last.1;
 
         // it is not allowed to use an iterator variable as assignee in a named expression
@@ -532,6 +622,7 @@ impl SymbolTableBuilder {
                 self.register_name("__module__", SymbolUsage::Assigned, location)?;
                 self.register_name("__qualname__", SymbolUsage::Assigned, location)?;
                 self.register_name("__doc__", SymbolUsage::Assigned, location)?;
+                self.register_name("__class__", SymbolUsage::Assigned, location)?;
                 self.scan_statements(body)?;
                 self.leave_scope();
                 self.scan_expressions(bases, ExpressionContext::Load)?;
@@ -954,7 +1045,7 @@ impl SymbolTableBuilder {
         let table = self.tables.last_mut().unwrap();
 
         // Some checks for the symbol that present on this scope level:
-        if let Some(symbol) = table.symbols.get(name) {
+        let symbol = if let Some(symbol) = table.symbols.get_mut(name) {
             // Role already set..
             match role {
                 SymbolUsage::Global => {
@@ -1000,6 +1091,7 @@ impl SymbolTableBuilder {
                     // Ok?
                 }
             }
+            symbol
         } else {
             // The symbol does not present on this scope level.
             // Some checks to insert new symbol into symbol table:
@@ -1016,11 +1108,10 @@ impl SymbolTableBuilder {
             }
             // Insert symbol when required:
             let symbol = Symbol::new(name);
-            table.symbols.insert(name.to_owned(), symbol);
-        }
+            table.symbols.entry(name.to_owned()).or_insert(symbol)
+        };
 
         // Set proper flags on symbol:
-        let symbol = table.symbols.get_mut(name).unwrap();
         match role {
             SymbolUsage::Nonlocal => {
                 symbol.scope = SymbolScope::Free;
@@ -1064,7 +1155,6 @@ impl SymbolTableBuilder {
             }
             SymbolUsage::Iter => {
                 symbol.is_iter = true;
-                symbol.scope = SymbolScope::Local;
             }
         }
 

From 8458e13e352e8c12dde38a0bcc2fbf4e7dbdcb8f Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 30 Nov 2020 19:40:12 -0600
Subject: [PATCH 170/459] Fast locals part 3

---
 src/compile.rs     |  64 ++++++++++++--------
 src/error.rs       |   4 ++
 src/symboltable.rs | 147 ++++++++++++++++++++++++++++++---------------
 3 files changed, 141 insertions(+), 74 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 2917ccd4..051cb2cd 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -273,14 +273,13 @@ impl Compiler {
         let cellvar_cache = table
             .symbols
             .iter()
-            .filter(|(_, s)| matches!(s.scope, SymbolScope::Cell))
+            .filter(|(_, s)| s.scope == SymbolScope::Cell)
             .map(|(var, _)| var.clone())
             .collect();
         let freevar_cache = table
             .symbols
             .iter()
-            // TODO: check if Free or FREE_CLASS symbol
-            .filter(|(_, s)| matches!(s.scope, SymbolScope::Free))
+            .filter(|(_, s)| s.scope == SymbolScope::Free || s.is_free_class)
             .map(|(var, _)| var.clone())
             .collect();
 
@@ -336,6 +335,11 @@ impl Compiler {
             let doc = self.name("__doc__");
             self.emit(Instruction::StoreGlobal(doc))
         }
+
+        if self.find_ann(statements) {
+            self.emit(Instruction::SetupAnnotation);
+        }
+
         self.compile_statements(statements)?;
 
         assert_eq!(self.code_stack.len(), size_before);
@@ -433,9 +437,12 @@ impl Compiler {
                 cache = &mut info.varname_cache;
                 NameOpType::Fast
             }
-            SymbolScope::GlobalImplicit if self.ctx.in_func() => NameOpType::Global,
-            SymbolScope::Local | SymbolScope::GlobalImplicit => NameOpType::Local,
             SymbolScope::GlobalExplicit => NameOpType::Global,
+            SymbolScope::GlobalImplicit | SymbolScope::Unknown if self.ctx.in_func() => {
+                NameOpType::Global
+            }
+            SymbolScope::GlobalImplicit | SymbolScope::Unknown => NameOpType::Local,
+            SymbolScope::Local => NameOpType::Local,
             SymbolScope::Free => {
                 cache = &mut info.freevar_cache;
                 NameOpType::Deref
@@ -444,8 +451,8 @@ impl Compiler {
                 cache = &mut info.cellvar_cache;
                 NameOpType::Deref
             }
-            // TODO: is this right?
-            SymbolScope::Unknown => NameOpType::Global,
+            // // TODO: is this right?
+            // SymbolScope::Unknown => NameOpType::Global,
         };
         let mut idx = cache
             .get_index_of(name)
@@ -527,6 +534,10 @@ impl Compiler {
                 let module_idx = module.as_ref().map(|s| self.name(s));
 
                 if import_star {
+                    if self.ctx.in_func() {
+                        return Err(self
+                            .error_loc(CompileErrorType::FunctionImportStar, statement.location));
+                    }
                     let star = self.name("*");
                     // from .... import *
                     self.emit(Instruction::Import {
@@ -1069,6 +1080,8 @@ impl Compiler {
         }
 
         let mut code = self.pop_code_object();
+        self.current_qualified_path = old_qualified_path;
+        self.ctx = prev_ctx;
 
         // Prepare type annotations:
         let mut num_annotations = 0;
@@ -1138,9 +1151,6 @@ impl Compiler {
         let doc = self.name("__doc__");
         self.emit(Instruction::StoreAttr { idx: doc });
 
-        self.current_qualified_path = old_qualified_path;
-        self.ctx = prev_ctx;
-
         self.apply_decorators(decorator_list);
 
         self.store_name(name);
@@ -1151,12 +1161,17 @@ impl Compiler {
     fn build_closure(&mut self, code: &CodeObject) {
         if !code.freevars.is_empty() {
             for var in &code.freevars {
-                let symbol = self.symbol_table_stack.last().unwrap().lookup(var).unwrap();
+                let table = self.symbol_table_stack.last().unwrap();
+                let symbol = table.lookup(var).unwrap();
                 let parent_code = self.code_stack.last().unwrap();
                 let vars = match symbol.scope {
                     SymbolScope::Free => &parent_code.freevar_cache,
                     SymbolScope::Cell => &parent_code.cellvar_cache,
-                    _ => unreachable!(),
+                    _ if symbol.is_free_class => &parent_code.freevar_cache,
+                    x => unreachable!(
+                        "var {} in a {:?} should be free or cell but it's {:?}",
+                        var, table.typ, x
+                    ),
                 };
                 let mut idx = vars.get_index_of(var).unwrap();
                 if let SymbolScope::Free = symbol.scope {
@@ -1236,6 +1251,8 @@ impl Compiler {
         keywords: &[ast::Keyword],
         decorator_list: &[ast::Expression],
     ) -> CompileResult<()> {
+        self.prepare_decorators(decorator_list)?;
+
         let prev_ctx = self.ctx;
         self.ctx = CompileContext {
             func: FunctionContext::NoFunction,
@@ -1247,7 +1264,6 @@ impl Compiler {
         let old_qualified_path = self.current_qualified_path.take();
         self.current_qualified_path = Some(qualified_name.clone());
 
-        self.prepare_decorators(decorator_list)?;
         self.emit(Instruction::LoadBuildClass);
         let line_number = self.get_source_line_number();
         self.push_output(CodeObject::new(
@@ -1301,6 +1317,9 @@ impl Compiler {
 
         let code = self.pop_code_object();
 
+        self.current_qualified_path = old_qualified_path;
+        self.ctx = prev_ctx;
+
         self.build_closure(&code);
 
         self.emit_constant(bytecode::ConstantData::Code {
@@ -1350,8 +1369,6 @@ impl Compiler {
         self.apply_decorators(decorator_list);
 
         self.store_name(name);
-        self.current_qualified_path = old_qualified_path;
-        self.ctx = prev_ctx;
         Ok(())
     }
 
@@ -1579,18 +1596,17 @@ impl Compiler {
 
         if let ast::ExpressionType::Identifier { name } = &target.node {
             // Store as dict entry in __annotations__ dict:
-            if !self.ctx.in_func() {
-                let annotations = self.name("__annotations__");
-                self.emit(Instruction::LoadNameAny(annotations));
-                self.emit_constant(bytecode::ConstantData::Str {
-                    value: name.to_owned(),
-                });
-                self.emit(Instruction::StoreSubscript);
-            }
+            let annotations = self.name("__annotations__");
+            self.emit(Instruction::LoadNameAny(annotations));
+            self.emit_constant(bytecode::ConstantData::Str {
+                value: name.to_owned(),
+            });
+            self.emit(Instruction::StoreSubscript);
         } else {
             // Drop annotation if not assigned to simple identifier.
             self.emit(Instruction::Pop);
         }
+
         Ok(())
     }
 
@@ -2187,7 +2203,7 @@ impl Compiler {
         let line_number = self.get_source_line_number();
         // Create magnificent function <listcomp>:
         self.push_output(CodeObject::new(
-            Default::default(),
+            bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED,
             1,
             1,
             0,
diff --git a/src/error.rs b/src/error.rs
index 37881f5c..b5b1d866 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -36,6 +36,7 @@ pub enum CompileErrorType {
     AsyncReturnValue,
     InvalidFuturePlacement,
     InvalidFutureFeature(String),
+    FunctionImportStar,
 }
 
 impl fmt::Display for CompileErrorType {
@@ -66,6 +67,9 @@ impl fmt::Display for CompileErrorType {
             CompileErrorType::InvalidFutureFeature(feat) => {
                 write!(f, "future feature {} is not defined", feat)
             }
+            CompileErrorType::FunctionImportStar => {
+                write!(f, "import * only allowed at module level")
+            }
         }
     }
 }
diff --git a/src/symboltable.rs b/src/symboltable.rs
index ce706208..ca1ed1af 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -85,7 +85,7 @@ impl fmt::Display for SymbolTableType {
 
 /// Indicator for a single symbol what the scope of this symbol is.
 /// The scope can be unknown, which is unfortunate, but not impossible.
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq)]
 pub enum SymbolScope {
     Unknown,
     Local,
@@ -116,6 +116,16 @@ pub struct Symbol {
     // inidicates that the symbol is used a bound iterator variable. We distinguish this case
     // from normal assignment to detect unallowed re-assignment to iterator variables.
     pub is_iter: bool,
+
+    /// indicates that the symbol is a free variable in a class method from the scope that the
+    /// class is defined in, e.g.:
+    /// ```python
+    /// def foo(x):
+    ///     class A:
+    ///         def method(self):
+    ///             return x // is_free_class
+    /// ```
+    pub is_free_class: bool,
 }
 
 impl Symbol {
@@ -131,6 +141,7 @@ impl Symbol {
             is_imported: false,
             is_assign_namedexpr_in_comprehension: false,
             is_iter: false,
+            is_free_class: false,
         }
     }
 
@@ -142,7 +153,7 @@ impl Symbol {
     }
 
     pub fn is_local(&self) -> bool {
-        matches!(self.scope, SymbolScope::Local)
+        self.scope == SymbolScope::Local
     }
 
     pub fn is_bound(&self) -> bool {
@@ -283,12 +294,10 @@ impl SymbolTableAnalyzer {
     fn analyze_symbol(
         &mut self,
         symbol: &mut Symbol,
-        curr_st_typ: SymbolTableType,
+        st_typ: SymbolTableType,
         sub_tables: &mut [SymbolTable],
     ) -> SymbolTableResult {
-        if symbol.is_assign_namedexpr_in_comprehension
-            && curr_st_typ == SymbolTableType::Comprehension
-        {
+        if symbol.is_assign_namedexpr_in_comprehension && st_typ == SymbolTableType::Comprehension {
             // propagate symbol to next higher level that can hold it,
             // i.e., function or module. Comprehension is skipped and
             // Class is not allowed and detected as error.
@@ -298,10 +307,12 @@ impl SymbolTableAnalyzer {
             match symbol.scope {
                 SymbolScope::Free => {
                     if !self.tables.as_ref().is_empty() {
-                        let scope_depth = self.tables.as_ref().iter().count();
+                        let scope_depth = self.tables.as_ref().len();
                         // check if the name is already defined in any outer scope
                         // therefore
-                        if scope_depth < 2 || !self.found_in_outer_scope(&symbol.name) {
+                        if scope_depth < 2
+                            || self.found_in_outer_scope(&symbol.name) != Some(SymbolScope::Free)
+                        {
                             return Err(SymbolTableError {
                                 error: format!("no binding for nonlocal '{}' found", symbol.name),
                                 // TODO: accurate location info, somehow
@@ -327,68 +338,101 @@ impl SymbolTableAnalyzer {
                 }
                 SymbolScope::Unknown => {
                     // Try hard to figure out what the scope of this symbol is.
-                    self.analyze_unknown_symbol(sub_tables, symbol);
+                    let scope = if symbol.is_bound() {
+                        self.found_in_inner_scope(sub_tables, &symbol.name, st_typ)
+                            .unwrap_or(SymbolScope::Local)
+                    } else if let Some(scope) = self.found_in_outer_scope(&symbol.name) {
+                        scope
+                    } else if self.tables.is_empty() {
+                        // Don't make assumptions when we don't know.
+                        SymbolScope::Unknown
+                    } else {
+                        // If there are scopes above we assume global.
+                        SymbolScope::GlobalImplicit
+                    };
+                    symbol.scope = scope;
                 }
             }
         }
         Ok(())
     }
 
-    fn found_in_outer_scope(&mut self, name: &str) -> bool {
+    fn found_in_outer_scope(&mut self, name: &str) -> Option<SymbolScope> {
         // Interesting stuff about the __class__ variable:
         // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
         if name == "__class__" {
-            return true;
+            return Some(SymbolScope::Free);
+        }
+
+        let mut decl_depth = None;
+        for (i, (symbols, typ)) in self.tables.iter().rev().enumerate() {
+            if let SymbolTableType::Class | SymbolTableType::Module = typ {
+                continue;
+            }
+            if let Some(sym) = symbols.get(name) {
+                match sym.scope {
+                    SymbolScope::GlobalExplicit => return Some(SymbolScope::GlobalExplicit),
+                    SymbolScope::GlobalImplicit => {}
+                    _ => {
+                        if sym.is_bound() {
+                            decl_depth = Some(i);
+                            break;
+                        }
+                    }
+                }
+            }
         }
-        let decl_depth = self.tables.iter().rev().position(|(symbols, typ)| {
-            !matches!(typ, SymbolTableType::Class | SymbolTableType::Module)
-                && symbols.get(name).map_or(false, |sym| sym.is_bound())
-        });
 
         if let Some(decl_depth) = decl_depth {
             // decl_depth is the number of tables between the current one and
             // the one that declared the cell var
-            for (table, _) in self.tables.iter_mut().rev().take(decl_depth) {
-                if !table.contains_key(name) {
+            for (table, typ) in self.tables.iter_mut().rev().take(decl_depth) {
+                if let SymbolTableType::Class = typ {
+                    if let Some(free_class) = table.get_mut(name) {
+                        free_class.is_free_class = true;
+                    } else {
+                        let mut symbol = Symbol::new(name);
+                        symbol.is_free_class = true;
+                        symbol.scope = SymbolScope::Free;
+                        table.insert(name.to_owned(), symbol);
+                    }
+                } else if !table.contains_key(name) {
                     let mut symbol = Symbol::new(name);
                     symbol.scope = SymbolScope::Free;
-                    symbol.is_referenced = true;
+                    // symbol.is_referenced = true;
                     table.insert(name.to_owned(), symbol);
                 }
             }
         }
 
-        decl_depth.is_some()
+        decl_depth.map(|_| SymbolScope::Free)
     }
 
-    fn found_in_inner_scope(sub_tables: &mut [SymbolTable], name: &str) -> bool {
-        sub_tables.iter().any(|x| {
-            x.symbols
-                .get(name)
-                .map_or(false, |sym| matches!(sym.scope, SymbolScope::Free))
+    fn found_in_inner_scope(
+        &self,
+        sub_tables: &[SymbolTable],
+        name: &str,
+        st_typ: SymbolTableType,
+    ) -> Option<SymbolScope> {
+        sub_tables.iter().find_map(|st| {
+            st.symbols.get(name).and_then(|sym| {
+                if sym.scope == SymbolScope::Free || sym.is_free_class {
+                    if st_typ == SymbolTableType::Class && name != "__class__" {
+                        None
+                    } else {
+                        Some(SymbolScope::Cell)
+                    }
+                } else if sym.scope == SymbolScope::GlobalExplicit && self.tables.is_empty() {
+                    // the symbol is defined on the module level, and an inner scope declares
+                    // a global that points to it
+                    Some(SymbolScope::GlobalExplicit)
+                } else {
+                    None
+                }
+            })
         })
     }
 
-    fn analyze_unknown_symbol(&mut self, sub_tables: &mut [SymbolTable], symbol: &mut Symbol) {
-        let scope = if symbol.is_bound() {
-            if Self::found_in_inner_scope(sub_tables, &symbol.name) {
-                SymbolScope::Cell
-            } else {
-                SymbolScope::Local
-            }
-        } else if self.found_in_outer_scope(&symbol.name) {
-            // Symbol is in some outer scope.
-            SymbolScope::Free
-        } else if self.tables.is_empty() {
-            // Don't make assumptions when we don't know.
-            SymbolScope::Unknown
-        } else {
-            // If there are scopes above we assume global.
-            SymbolScope::GlobalImplicit
-        };
-        symbol.scope = scope;
-    }
-
     // Implements the symbol analysis and scope extension for names
     // assigned by a named expression in a comprehension. See:
     // https://github.com/python/cpython/blob/7b78e7f9fd77bb3280ee39fb74b86772a7d46a70/Python/symtable.c#L1435
@@ -431,12 +475,13 @@ impl SymbolTableAnalyzer {
                     if let SymbolScope::Unknown = parent_symbol.scope {
                         // this information is new, as the asignment is done in inner scope
                         parent_symbol.is_assigned = true;
-                        //self.analyze_unknown_symbol(symbol); // not needed, symbol is analyzed anyhow when its scope is analyzed
                     }
 
-                    if !symbol.is_global() {
-                        symbol.scope = SymbolScope::Free;
-                    }
+                    symbol.scope = if parent_symbol.is_global() {
+                        parent_symbol.scope
+                    } else {
+                        SymbolScope::Free
+                    };
                 } else {
                     let mut cloned_sym = symbol.clone();
                     cloned_sym.scope = SymbolScope::Local;
@@ -907,6 +952,7 @@ impl SymbolTableBuilder {
                 // Determine the contextual usage of this symbol:
                 match context {
                     ExpressionContext::Delete => {
+                        self.register_name(name, SymbolUsage::Assigned, location)?;
                         self.register_name(name, SymbolUsage::Used, location)?;
                     }
                     ExpressionContext::Load | ExpressionContext::IterDefinitionExp => {
@@ -1051,6 +1097,7 @@ impl SymbolTableBuilder {
                 SymbolUsage::Global => {
                     if symbol.is_global() {
                         // Ok
+                        symbol.scope = SymbolScope::GlobalExplicit;
                     } else {
                         return Err(SymbolTableError {
                             error: format!("name '{}' is used prior to global declaration", name),
@@ -1140,9 +1187,9 @@ impl SymbolTableBuilder {
             }
             SymbolUsage::Global => {
                 if let SymbolScope::Unknown = symbol.scope {
-                    symbol.scope = SymbolScope::GlobalImplicit;
+                    symbol.scope = SymbolScope::GlobalExplicit;
                 } else if symbol.is_global() {
-                    // Global scope can be set to global
+                    symbol.scope = SymbolScope::GlobalExplicit;
                 } else {
                     return Err(SymbolTableError {
                         error: format!("Symbol {} scope cannot be set to global, since its scope was already determined otherwise.", name),

From 9305316f6907d44d31fe640a27419f6960c2f04b Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 30 Nov 2020 22:35:02 -0600
Subject: [PATCH 171/459] Use a boxed slice for codeobject fields

---
 src/compile.rs | 63 ++++++++++++++++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 23 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 051cb2cd..32441d5d 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -18,6 +18,9 @@ type CompileResult<T> = Result<T, CompileError>;
 
 struct CodeInfo {
     code: CodeObject,
+    instructions: Vec<Instruction>,
+    locations: Vec<bytecode::Location>,
+    constants: Vec<bytecode::ConstantData>,
     name_cache: IndexSet<String>,
     varname_cache: IndexSet<String>,
     cellvar_cache: IndexSet<String>,
@@ -28,6 +31,9 @@ impl CodeInfo {
     fn finalize_code(self) -> CodeObject {
         let CodeInfo {
             mut code,
+            instructions,
+            locations,
+            constants,
             name_cache,
             varname_cache,
             cellvar_cache,
@@ -35,10 +41,13 @@ impl CodeInfo {
             label_map,
         } = self;
 
-        code.names.extend(name_cache);
-        code.varnames.extend(varname_cache);
-        code.cellvars.extend(cellvar_cache);
-        code.freevars.extend(freevar_cache);
+        code.instructions = instructions.into();
+        code.locations = locations.into();
+        code.constants = constants.into();
+        code.names = name_cache.into_iter().collect();
+        code.varnames = varname_cache.into_iter().collect();
+        code.cellvars = cellvar_cache.into_iter().collect();
+        code.freevars = freevar_cache.into_iter().collect();
 
         if !code.cellvars.is_empty() {
             let total_args = code.arg_count
@@ -65,7 +74,7 @@ impl CodeInfo {
             }
         }
 
-        for instruction in &mut code.instructions {
+        for instruction in &mut *code.instructions {
             use Instruction::*;
             // this is a little bit hacky, as until now the data stored inside Labels in
             // Instructions is just bookkeeping, but I think it's the best way to do this
@@ -221,7 +230,7 @@ impl Compiler {
     fn new(opts: CompileOpts, source_path: String, code_name: String) -> Self {
         let module_code = CodeInfo {
             code: CodeObject::new(
-                Default::default(),
+                bytecode::CodeFlags::NEW_LOCALS,
                 0,
                 0,
                 0,
@@ -229,6 +238,9 @@ impl Compiler {
                 0,
                 code_name,
             ),
+            instructions: Vec::new(),
+            locations: Vec::new(),
+            constants: Vec::new(),
             name_cache: IndexSet::new(),
             varname_cache: IndexSet::new(),
             cellvar_cache: IndexSet::new(),
@@ -287,6 +299,9 @@ impl Compiler {
 
         let info = CodeInfo {
             code,
+            instructions: Vec::new(),
+            locations: Vec::new(),
+            constants: Vec::new(),
             name_cache: IndexSet::new(),
             varname_cache: IndexSet::new(),
             cellvar_cache,
@@ -315,7 +330,7 @@ impl Compiler {
         name: &str,
         cache: impl FnOnce(&mut CodeInfo) -> &mut IndexSet<String>,
     ) -> bytecode::NameIdx {
-        let cache = cache(self.code_stack.last_mut().expect("nothing on stack"));
+        let cache = cache(self.current_codeinfo());
         cache
             .get_index_of(name)
             .unwrap_or_else(|| cache.insert_full(name.to_owned()).0)
@@ -1160,7 +1175,7 @@ impl Compiler {
 
     fn build_closure(&mut self, code: &CodeObject) {
         if !code.freevars.is_empty() {
-            for var in &code.freevars {
+            for var in &*code.freevars {
                 let table = self.symbol_table_stack.last().unwrap();
                 let symbol = table.lookup(var).unwrap();
                 let parent_code = self.code_stack.last().unwrap();
@@ -2423,29 +2438,29 @@ impl Compiler {
     fn emit(&mut self, instruction: Instruction) {
         let location = compile_location(&self.current_source_location);
         // TODO: insert source filename
-        let code = self.current_code();
-        code.instructions.push(instruction);
-        code.locations.push(location);
+        let info = self.current_codeinfo();
+        info.instructions.push(instruction);
+        info.locations.push(location);
     }
 
     fn emit_constant(&mut self, constant: bytecode::ConstantData) {
-        let code = self.current_code();
-        let idx = code.constants.len();
-        code.constants.push(constant);
+        let info = self.current_codeinfo();
+        let idx = info.constants.len();
+        info.constants.push(constant);
         self.emit(Instruction::LoadConst { idx })
     }
 
     fn current_code(&mut self) -> &mut CodeObject {
-        &mut self
-            .code_stack
-            .last_mut()
-            .expect("No OutputStream on stack")
-            .code
+        &mut self.current_codeinfo().code
+    }
+
+    fn current_codeinfo(&mut self) -> &mut CodeInfo {
+        self.code_stack.last_mut().expect("no code on stack")
     }
 
     // Generate a new label
     fn new_label(&mut self) -> Label {
-        let label_map = &mut self.code_stack.last_mut().unwrap().label_map;
+        let label_map = &mut self.current_codeinfo().label_map;
         let label = Label(label_map.len());
         label_map.push(None);
         label
@@ -2454,9 +2469,11 @@ impl Compiler {
     // Assign current position the given label
     fn set_label(&mut self, label: Label) {
         let CodeInfo {
-            code, label_map, ..
-        } = self.code_stack.last_mut().unwrap();
-        let actual_label = Label(code.instructions.len());
+            instructions,
+            label_map,
+            ..
+        } = self.current_codeinfo();
+        let actual_label = Label(instructions.len());
         let prev_val = std::mem::replace(&mut label_map[label.0], Some(actual_label));
         debug_assert!(
             prev_val.map_or(true, |x| x == actual_label),

From ff8512cd54043312d281b263e744438dd7aa8a99 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 4 Dec 2020 14:18:17 -0600
Subject: [PATCH 172/459] Apply review suggestions

---
 src/compile.rs     | 11 ++++-------
 src/symboltable.rs |  9 +++++++--
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 32441d5d..da500a60 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -60,13 +60,10 @@ impl CodeInfo {
                 .cellvars
                 .iter()
                 .map(|var| {
-                    for (i, arg) in all_args.iter().enumerate() {
-                        if var == arg {
-                            found_cellarg = true;
-                            return i as isize;
-                        }
-                    }
-                    -1
+                    all_args.iter().position(|arg| var == arg).map_or(-1, |i| {
+                        found_cellarg = true;
+                        i as isize
+                    })
                 })
                 .collect::<Box<[_]>>();
             if found_cellarg {
diff --git a/src/symboltable.rs b/src/symboltable.rs
index ca1ed1af..e5c238ea 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -108,6 +108,7 @@ pub struct Symbol {
     pub is_parameter: bool,
     pub is_annotated: bool,
     pub is_imported: bool,
+    pub is_nonlocal: bool,
 
     // indicates if the symbol gets a value assigned by a named expression in a comprehension
     // this is required to correct the scope in the analysis.
@@ -139,6 +140,7 @@ impl Symbol {
             is_parameter: false,
             is_annotated: false,
             is_imported: false,
+            is_nonlocal: false,
             is_assign_namedexpr_in_comprehension: false,
             is_iter: false,
             is_free_class: false,
@@ -218,7 +220,9 @@ mod stack {
         }
     }
     impl<T> StackStack<T> {
-        pub fn append<F, R>(&mut self, x: &mut T, f: F) -> R
+        /// Appends a reference to this stack for the duration of the function `f`. When `f`
+        /// returns, the reference will be popped off the stack.
+        pub fn with_append<F, R>(&mut self, x: &mut T, f: F) -> R
         where
             F: FnOnce(&mut Self) -> R,
         {
@@ -273,7 +277,7 @@ impl SymbolTableAnalyzer {
         let sub_tables = &mut *symbol_table.sub_tables;
 
         let mut info = (symbols, symbol_table.typ);
-        self.tables.append(&mut info, |list| {
+        self.tables.with_append(&mut info, |list| {
             let inner_scope = unsafe { &mut *(list as *mut _ as *mut SymbolTableAnalyzer) };
             // Analyze sub scopes:
             for sub_table in sub_tables.iter_mut() {
@@ -1162,6 +1166,7 @@ impl SymbolTableBuilder {
         match role {
             SymbolUsage::Nonlocal => {
                 symbol.scope = SymbolScope::Free;
+                symbol.is_nonlocal = true;
             }
             SymbolUsage::Imported => {
                 symbol.is_assigned = true;

From 281e95566360b8e8abc47e871aa999a59ebd98a0 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Thu, 10 Dec 2020 12:36:29 -0600
Subject: [PATCH 173/459] Import ConstantData in compile.rs

---
 src/compile.rs | 94 +++++++++++++++++++++++++-------------------------
 1 file changed, 47 insertions(+), 47 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index da500a60..7204ff3f 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -12,7 +12,7 @@ use indexmap::IndexSet;
 use itertools::Itertools;
 use num_complex::Complex64;
 use rustpython_ast as ast;
-use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Label};
+use rustpython_bytecode::bytecode::{self, CallType, CodeObject, ConstantData, Instruction, Label};
 
 type CompileResult<T> = Result<T, CompileError>;
 
@@ -20,7 +20,7 @@ struct CodeInfo {
     code: CodeObject,
     instructions: Vec<Instruction>,
     locations: Vec<bytecode::Location>,
-    constants: Vec<bytecode::ConstantData>,
+    constants: Vec<ConstantData>,
     name_cache: IndexSet<String>,
     varname_cache: IndexSet<String>,
     cellvar_cache: IndexSet<String>,
@@ -343,7 +343,7 @@ impl Compiler {
 
         let (statements, doc) = get_doc(&program.statements);
         if let Some(value) = doc {
-            self.emit_constant(bytecode::ConstantData::Str { value });
+            self.emit_constant(ConstantData::Str { value });
             let doc = self.name("__doc__");
             self.emit(Instruction::StoreGlobal(doc))
         }
@@ -357,7 +357,7 @@ impl Compiler {
         assert_eq!(self.code_stack.len(), size_before);
 
         // Emit None at end:
-        self.emit_constant(bytecode::ConstantData::None);
+        self.emit_constant(ConstantData::None);
         self.emit(Instruction::ReturnValue);
         Ok(())
     }
@@ -391,7 +391,7 @@ impl Compiler {
         }
 
         if !emitted_return {
-            self.emit_constant(bytecode::ConstantData::None);
+            self.emit_constant(ConstantData::None);
             self.emit(Instruction::ReturnValue);
         }
 
@@ -636,7 +636,7 @@ impl Compiler {
                         if is_async {
                             self.emit(Instruction::BeforeAsyncWith);
                             self.emit(Instruction::GetAwaitable);
-                            self.emit_constant(bytecode::ConstantData::None);
+                            self.emit_constant(ConstantData::None);
                             self.emit(Instruction::YieldFrom);
                             self.emit(Instruction::SetupAsyncWith { end: end_label });
                         } else {
@@ -667,7 +667,7 @@ impl Compiler {
 
                     if is_async {
                         self.emit(Instruction::GetAwaitable);
-                        self.emit_constant(bytecode::ConstantData::None);
+                        self.emit_constant(ConstantData::None);
                         self.emit(Instruction::YieldFrom);
                     }
 
@@ -779,7 +779,7 @@ impl Compiler {
                         self.compile_expression(v)?;
                     }
                     None => {
-                        self.emit_constant(bytecode::ConstantData::None);
+                        self.emit_constant(ConstantData::None);
                     }
                 }
 
@@ -862,7 +862,7 @@ impl Compiler {
         let mut num_kw_only_defaults = 0;
         for (kw, default) in args.kwonlyargs.iter().zip(&args.kw_defaults) {
             if let Some(default) = default {
-                self.emit_constant(bytecode::ConstantData::Str {
+                self.emit_constant(ConstantData::Str {
                     value: kw.arg.clone(),
                 });
                 self.compile_expression(default)?;
@@ -1086,7 +1086,7 @@ impl Compiler {
                 // the last instruction is a ReturnValue already, we don't need to emit it
             }
             _ => {
-                self.emit_constant(bytecode::ConstantData::None);
+                self.emit_constant(ConstantData::None);
                 self.emit(Instruction::ReturnValue);
             }
         }
@@ -1101,7 +1101,7 @@ impl Compiler {
         // Return annotation:
         if let Some(annotation) = returns {
             // key:
-            self.emit_constant(bytecode::ConstantData::Str {
+            self.emit_constant(ConstantData::Str {
                 value: "return".to_owned(),
             });
             // value:
@@ -1111,7 +1111,7 @@ impl Compiler {
 
         let mut visit_arg_annotation = |arg: &ast::Parameter| -> CompileResult<()> {
             if let Some(annotation) = &arg.annotation {
-                self.emit_constant(bytecode::ConstantData::Str {
+                self.emit_constant(ConstantData::Str {
                     value: arg.arg.to_owned(),
                 });
                 self.compile_expression(&annotation)?;
@@ -1147,10 +1147,10 @@ impl Compiler {
 
         self.build_closure(&code);
 
-        self.emit_constant(bytecode::ConstantData::Code {
+        self.emit_constant(ConstantData::Code {
             code: Box::new(code),
         });
-        self.emit_constant(bytecode::ConstantData::Str {
+        self.emit_constant(ConstantData::Str {
             value: qualified_name,
         });
 
@@ -1294,7 +1294,7 @@ impl Compiler {
         self.emit(Instruction::LoadGlobal(dunder_name));
         let dunder_module = self.name("__module__");
         self.emit(Instruction::StoreLocal(dunder_module));
-        self.emit_constant(bytecode::ConstantData::Str {
+        self.emit_constant(ConstantData::Str {
             value: qualified_name.clone(),
         });
         let qualname = self.name("__qualname__");
@@ -1322,7 +1322,7 @@ impl Compiler {
             let classcell = self.name("__classcell__");
             self.emit(Instruction::StoreLocal(classcell));
         } else {
-            self.emit_constant(bytecode::ConstantData::None);
+            self.emit_constant(ConstantData::None);
         }
 
         self.emit(Instruction::ReturnValue);
@@ -1334,17 +1334,17 @@ impl Compiler {
 
         self.build_closure(&code);
 
-        self.emit_constant(bytecode::ConstantData::Code {
+        self.emit_constant(ConstantData::Code {
             code: Box::new(code),
         });
-        self.emit_constant(bytecode::ConstantData::Str {
+        self.emit_constant(ConstantData::Str {
             value: name.to_owned(),
         });
 
         // Turn code object into function object:
         self.emit(Instruction::MakeFunction);
 
-        self.emit_constant(bytecode::ConstantData::Str {
+        self.emit_constant(ConstantData::Str {
             value: qualified_name,
         });
 
@@ -1356,7 +1356,7 @@ impl Compiler {
             let mut kwarg_names = vec![];
             for keyword in keywords {
                 if let Some(name) = &keyword.name {
-                    kwarg_names.push(bytecode::ConstantData::Str {
+                    kwarg_names.push(ConstantData::Str {
                         value: name.to_owned(),
                     });
                 } else {
@@ -1366,7 +1366,7 @@ impl Compiler {
                 self.compile_expression(&keyword.value)?;
             }
 
-            self.emit_constant(bytecode::ConstantData::Tuple {
+            self.emit_constant(ConstantData::Tuple {
                 elements: kwarg_names,
             });
             self.emit(Instruction::CallFunction {
@@ -1390,8 +1390,8 @@ impl Compiler {
 
         // Doc string value:
         self.emit_constant(match doc_str {
-            Some(doc) => bytecode::ConstantData::Str { value: doc },
-            None => bytecode::ConstantData::None, // set docstring None if not declared
+            Some(doc) => ConstantData::Str { value: doc },
+            None => ConstantData::None, // set docstring None if not declared
         });
     }
 
@@ -1461,7 +1461,7 @@ impl Compiler {
                 handler: check_asynciter_label,
             });
             self.emit(Instruction::GetANext);
-            self.emit_constant(bytecode::ConstantData::None);
+            self.emit_constant(ConstantData::None);
             self.emit(Instruction::YieldFrom);
             self.compile_store(target)?;
             self.emit(Instruction::PopBlock);
@@ -1610,7 +1610,7 @@ impl Compiler {
             // Store as dict entry in __annotations__ dict:
             let annotations = self.name("__annotations__");
             self.emit(Instruction::LoadNameAny(annotations));
-            self.emit_constant(bytecode::ConstantData::Str {
+            self.emit_constant(ConstantData::Str {
                 value: name.to_owned(),
             });
             self.emit(Instruction::StoreSubscript);
@@ -1909,11 +1909,11 @@ impl Compiler {
             }
             Number { value } => {
                 let const_value = match value {
-                    ast::Number::Integer { value } => bytecode::ConstantData::Integer {
+                    ast::Number::Integer { value } => ConstantData::Integer {
                         value: value.clone(),
                     },
-                    ast::Number::Float { value } => bytecode::ConstantData::Float { value: *value },
-                    ast::Number::Complex { real, imag } => bytecode::ConstantData::Complex {
+                    ast::Number::Float { value } => ConstantData::Float { value: *value },
+                    ast::Number::Complex { real, imag } => ConstantData::Complex {
                         value: Complex64::new(*real, *imag),
                     },
                 };
@@ -1960,7 +1960,7 @@ impl Compiler {
                 self.mark_generator();
                 match value {
                     Some(expression) => self.compile_expression(expression)?,
-                    Option::None => self.emit_constant(bytecode::ConstantData::None),
+                    Option::None => self.emit_constant(ConstantData::None),
                 };
                 self.emit(Instruction::YieldValue);
             }
@@ -1970,7 +1970,7 @@ impl Compiler {
                 }
                 self.compile_expression(value)?;
                 self.emit(Instruction::GetAwaitable);
-                self.emit_constant(bytecode::ConstantData::None);
+                self.emit_constant(ConstantData::None);
                 self.emit(Instruction::YieldFrom);
             }
             YieldFrom { value } => {
@@ -1986,26 +1986,26 @@ impl Compiler {
                 self.mark_generator();
                 self.compile_expression(value)?;
                 self.emit(Instruction::GetIter);
-                self.emit_constant(bytecode::ConstantData::None);
+                self.emit_constant(ConstantData::None);
                 self.emit(Instruction::YieldFrom);
             }
             True => {
-                self.emit_constant(bytecode::ConstantData::Boolean { value: true });
+                self.emit_constant(ConstantData::Boolean { value: true });
             }
             False => {
-                self.emit_constant(bytecode::ConstantData::Boolean { value: false });
+                self.emit_constant(ConstantData::Boolean { value: false });
             }
             ast::ExpressionType::None => {
-                self.emit_constant(bytecode::ConstantData::None);
+                self.emit_constant(ConstantData::None);
             }
             Ellipsis => {
-                self.emit_constant(bytecode::ConstantData::Ellipsis);
+                self.emit_constant(ConstantData::Ellipsis);
             }
             ast::ExpressionType::String { value } => {
                 self.compile_string(value)?;
             }
             Bytes { value } => {
-                self.emit_constant(bytecode::ConstantData::Bytes {
+                self.emit_constant(ConstantData::Bytes {
                     value: value.clone(),
                 });
             }
@@ -2026,10 +2026,10 @@ impl Compiler {
                 self.emit(Instruction::ReturnValue);
                 let code = self.pop_code_object();
                 self.build_closure(&code);
-                self.emit_constant(bytecode::ConstantData::Code {
+                self.emit_constant(ConstantData::Code {
                     code: Box::new(code),
                 });
-                self.emit_constant(bytecode::ConstantData::Str { value: name });
+                self.emit_constant(ConstantData::Str { value: name });
                 // Turn code object into function object:
                 self.emit(Instruction::MakeFunction);
 
@@ -2076,7 +2076,7 @@ impl Compiler {
                 let mut subsize = 0;
                 for keyword in subkeywords {
                     if let Some(name) = &keyword.name {
-                        self.emit_constant(bytecode::ConstantData::Str {
+                        self.emit_constant(ConstantData::Str {
                             value: name.to_owned(),
                         });
                         self.compile_expression(&keyword.value)?;
@@ -2138,7 +2138,7 @@ impl Compiler {
                 let mut kwarg_names = vec![];
                 for keyword in keywords {
                     if let Some(name) = &keyword.name {
-                        kwarg_names.push(bytecode::ConstantData::Str {
+                        kwarg_names.push(ConstantData::Str {
                             value: name.to_owned(),
                         });
                     } else {
@@ -2148,7 +2148,7 @@ impl Compiler {
                     self.compile_expression(&keyword.value)?;
                 }
 
-                self.emit_constant(bytecode::ConstantData::Tuple {
+                self.emit_constant(ConstantData::Tuple {
                     elements: kwarg_names,
                 });
                 self.emit(Instruction::CallFunction {
@@ -2349,12 +2349,12 @@ impl Compiler {
         self.build_closure(&code);
 
         // List comprehension code:
-        self.emit_constant(bytecode::ConstantData::Code {
+        self.emit_constant(ConstantData::Code {
             code: Box::new(code),
         });
 
         // List comprehension function name:
-        self.emit_constant(bytecode::ConstantData::Str { value: name });
+        self.emit_constant(ConstantData::Str { value: name });
 
         // Turn code object into function object:
         self.emit(Instruction::MakeFunction);
@@ -2374,7 +2374,7 @@ impl Compiler {
 
     fn compile_string(&mut self, string: &ast::StringGroup) -> CompileResult<()> {
         if let Some(value) = try_get_constant_string(string) {
-            self.emit_constant(bytecode::ConstantData::Str { value });
+            self.emit_constant(ConstantData::Str { value });
         } else {
             match string {
                 ast::StringGroup::Joined { values } => {
@@ -2384,7 +2384,7 @@ impl Compiler {
                     self.emit(Instruction::BuildString { size: values.len() })
                 }
                 ast::StringGroup::Constant { value } => {
-                    self.emit_constant(bytecode::ConstantData::Str {
+                    self.emit_constant(ConstantData::Str {
                         value: value.to_owned(),
                     });
                 }
@@ -2395,7 +2395,7 @@ impl Compiler {
                 } => {
                     match spec {
                         Some(spec) => self.compile_string(spec)?,
-                        None => self.emit_constant(bytecode::ConstantData::Str {
+                        None => self.emit_constant(ConstantData::Str {
                             value: String::new(),
                         }),
                     };
@@ -2440,7 +2440,7 @@ impl Compiler {
         info.locations.push(location);
     }
 
-    fn emit_constant(&mut self, constant: bytecode::ConstantData) {
+    fn emit_constant(&mut self, constant: ConstantData) {
         let info = self.current_codeinfo();
         let idx = info.constants.len();
         info.constants.push(constant);

From 48ca658ba052c552a29cbb320aa977375ba785a7 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Thu, 10 Dec 2020 11:22:21 -0600
Subject: [PATCH 174/459] Optimize the size of Instruction

---
 Cargo.toml     |   1 +
 src/compile.rs | 163 ++++++++++++++++++++++---------------------------
 src/error.rs   |   4 ++
 3 files changed, 77 insertions(+), 91 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index dc44869a..ba37125e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,6 +13,7 @@ itertools = "0.9"
 rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
 rustpython-ast = { path = "../ast" }
 num-complex = { version = "0.3", features = ["serde"] }
+num-traits = "0.2"
 log = "0.4"
 arrayvec = "0.5"
 
diff --git a/src/compile.rs b/src/compile.rs
index 7204ff3f..e953ec83 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -11,8 +11,9 @@ use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolSc
 use indexmap::IndexSet;
 use itertools::Itertools;
 use num_complex::Complex64;
+use num_traits::ToPrimitive;
 use rustpython_ast as ast;
-use rustpython_bytecode::bytecode::{self, CallType, CodeObject, ConstantData, Instruction, Label};
+use rustpython_bytecode::bytecode::{self, CodeObject, ConstantData, Instruction, Label};
 
 type CompileResult<T> = Result<T, CompileError>;
 
@@ -86,29 +87,12 @@ impl CodeInfo {
                 | SetupFinally { handler: l }
                 | SetupExcept { handler: l }
                 | SetupWith { end: l }
-                | SetupAsyncWith { end: l } => {
+                | SetupAsyncWith { end: l }
+                | SetupLoop { end: l } => {
                     *l = label_map[l.0].expect("label never set");
                 }
-                SetupLoop { start, end } => {
-                    *start = label_map[start.0].expect("label never set");
-                    *end = label_map[end.0].expect("label never set");
-                }
 
-                #[rustfmt::skip]
-                Import { .. } | ImportStar | ImportFrom { .. } | LoadFast(_) | LoadNameAny(_)
-                | LoadGlobal(_) | LoadDeref(_) | LoadClassDeref(_) | StoreFast(_) | StoreLocal(_)
-                | StoreGlobal(_) | StoreDeref(_) | DeleteFast(_) | DeleteLocal(_) | DeleteGlobal(_)
-                | DeleteDeref(_) | LoadClosure(_) | Subscript | StoreSubscript | DeleteSubscript
-                | StoreAttr { .. } | DeleteAttr { .. } | LoadConst { .. } | UnaryOperation { .. }
-                | BinaryOperation { .. } | LoadAttr { .. } | CompareOperation { .. } | Pop
-                | Rotate { .. } | Duplicate | GetIter | Continue | Break | MakeFunction
-                | CallFunction { .. } | ReturnValue | YieldValue | YieldFrom | SetupAnnotation
-                | EnterFinally | EndFinally | WithCleanupStart | WithCleanupFinish | PopBlock
-                | Raise { .. } | BuildString { .. } | BuildTuple { .. } | BuildList { .. }
-                | BuildSet { .. } | BuildMap { .. } | BuildSlice { .. } | ListAppend { .. }
-                | SetAdd { .. } | MapAdd { .. } | PrintExpr | LoadBuildClass | UnpackSequence { .. }
-                | UnpackEx { .. } | FormatValue { .. } | PopException | Reverse { .. }
-                | GetAwaitable | BeforeAsyncWith | GetAIter | GetANext | MapAddRev { .. } => {}
+                _ => {}
             }
         }
         code
@@ -519,12 +503,12 @@ impl Compiler {
             Import { names } => {
                 // import a, b, c as d
                 for name in names {
-                    let name_idx = Some(self.name(&name.symbol));
-                    self.emit(Instruction::Import {
-                        name_idx,
-                        symbols_idx: vec![],
-                        level: 0,
+                    self.emit_constant(ConstantData::Integer {
+                        value: num_traits::Zero::zero(),
                     });
+                    self.emit_constant(ConstantData::None);
+                    let idx = self.name(&name.symbol);
+                    self.emit(Instruction::ImportName { idx });
                     if let Some(alias) = &name.alias {
                         for part in name.symbol.split('.').skip(1) {
                             let idx = self.name(part);
@@ -543,32 +527,43 @@ impl Compiler {
             } => {
                 let import_star = names.iter().any(|n| n.symbol == "*");
 
-                let module_idx = module.as_ref().map(|s| self.name(s));
-
-                if import_star {
+                let from_list = if import_star {
                     if self.ctx.in_func() {
                         return Err(self
                             .error_loc(CompileErrorType::FunctionImportStar, statement.location));
                     }
-                    let star = self.name("*");
+                    vec![ConstantData::Str {
+                        value: "*".to_owned(),
+                    }]
+                } else {
+                    names
+                        .iter()
+                        .map(|n| ConstantData::Str {
+                            value: n.symbol.to_owned(),
+                        })
+                        .collect()
+                };
+
+                let module_idx = module.as_ref().map(|s| self.name(s));
+
+                // from .... import (*fromlist)
+                self.emit_constant(ConstantData::Integer {
+                    value: (*level).into(),
+                });
+                self.emit_constant(ConstantData::Tuple {
+                    elements: from_list,
+                });
+                if let Some(idx) = module_idx {
+                    self.emit(Instruction::ImportName { idx });
+                } else {
+                    self.emit(Instruction::ImportNameless);
+                }
+
+                if import_star {
                     // from .... import *
-                    self.emit(Instruction::Import {
-                        name_idx: module_idx,
-                        symbols_idx: vec![star],
-                        level: *level,
-                    });
                     self.emit(Instruction::ImportStar);
                 } else {
                     // from mod import a, b as c
-                    // First, determine the fromlist (for import lib):
-                    let from_list = names.iter().map(|n| self.name(&n.symbol)).collect();
-
-                    // Load module once:
-                    self.emit(Instruction::Import {
-                        name_idx: module_idx,
-                        symbols_idx: from_list,
-                        level: *level,
-                    });
 
                     for name in names {
                         let idx = self.name(&name.symbol);
@@ -731,14 +726,10 @@ impl Compiler {
                     match msg {
                         Some(e) => {
                             self.compile_expression(e)?;
-                            self.emit(Instruction::CallFunction {
-                                typ: CallType::Positional(1),
-                            });
+                            self.emit(Instruction::CallFunctionPositional { nargs: 1 });
                         }
                         None => {
-                            self.emit(Instruction::CallFunction {
-                                typ: CallType::Positional(0),
-                            });
+                            self.emit(Instruction::CallFunctionPositional { nargs: 0 });
                         }
                     }
                     self.emit(Instruction::Raise { argc: 1 });
@@ -927,9 +918,7 @@ impl Compiler {
     fn apply_decorators(&mut self, decorator_list: &[ast::Expression]) {
         // Apply decorators:
         for _ in decorator_list {
-            self.emit(Instruction::CallFunction {
-                typ: CallType::Positional(1),
-            });
+            self.emit(Instruction::CallFunctionPositional { nargs: 1 });
         }
     }
 
@@ -1369,12 +1358,12 @@ impl Compiler {
             self.emit_constant(ConstantData::Tuple {
                 elements: kwarg_names,
             });
-            self.emit(Instruction::CallFunction {
-                typ: CallType::Keyword(2 + keywords.len() + bases.len()),
+            self.emit(Instruction::CallFunctionKeyword {
+                nargs: 2 + keywords.len() + bases.len(),
             });
         } else {
-            self.emit(Instruction::CallFunction {
-                typ: CallType::Positional(2 + bases.len()),
+            self.emit(Instruction::CallFunctionPositional {
+                nargs: 2 + bases.len(),
             });
         }
 
@@ -1404,11 +1393,8 @@ impl Compiler {
         let start_label = self.new_label();
         let else_label = self.new_label();
         let end_label = self.new_label();
-        self.emit(Instruction::SetupLoop {
-            start: start_label,
-            end: end_label,
-        });
 
+        self.emit(Instruction::SetupLoop { end: end_label });
         self.set_label(start_label);
 
         self.compile_jump_if(test, false, else_label)?;
@@ -1442,11 +1428,6 @@ impl Compiler {
         let else_label = self.new_label();
         let end_label = self.new_label();
 
-        self.emit(Instruction::SetupLoop {
-            start: start_label,
-            end: end_label,
-        });
-
         // The thing iterated:
         self.compile_expression(iter)?;
 
@@ -1456,6 +1437,7 @@ impl Compiler {
 
             self.emit(Instruction::GetAIter);
 
+            self.emit(Instruction::SetupLoop { end: end_label });
             self.set_label(start_label);
             self.emit(Instruction::SetupExcept {
                 handler: check_asynciter_label,
@@ -1486,6 +1468,7 @@ impl Compiler {
             // Retrieve Iterator
             self.emit(Instruction::GetIter);
 
+            self.emit(Instruction::SetupLoop { end: end_label });
             self.set_label(start_label);
             self.emit(Instruction::ForIter { target: else_label });
 
@@ -1647,10 +1630,16 @@ impl Compiler {
                             return Err(self.error(CompileErrorType::MultipleStarArgs));
                         } else {
                             seen_star = true;
-                            self.emit(Instruction::UnpackEx {
-                                before: i,
-                                after: elements.len() - i - 1,
-                            });
+                            let before = i;
+                            let after = elements.len() - i - 1;
+                            let (before, after) = (|| Some((before.to_u8()?, after.to_u8()?)))()
+                                .ok_or_else(|| {
+                                    self.error_loc(
+                                        CompileErrorType::TooManyStarUnpack,
+                                        target.location,
+                                    )
+                                })?;
+                            self.emit(Instruction::UnpackEx { before, after });
                         }
                     }
                 }
@@ -1683,7 +1672,7 @@ impl Compiler {
     }
 
     fn compile_op(&mut self, op: &ast::Operator, inplace: bool) {
-        let i = match op {
+        let op = match op {
             ast::Operator::Add => bytecode::BinaryOperator::Add,
             ast::Operator::Sub => bytecode::BinaryOperator::Subtract,
             ast::Operator::Mult => bytecode::BinaryOperator::Multiply,
@@ -1698,7 +1687,12 @@ impl Compiler {
             ast::Operator::BitXor => bytecode::BinaryOperator::Xor,
             ast::Operator::BitAnd => bytecode::BinaryOperator::And,
         };
-        self.emit(Instruction::BinaryOperation { op: i, inplace });
+        let ins = if inplace {
+            Instruction::BinaryOperationInplace { op }
+        } else {
+            Instruction::BinaryOperation { op }
+        };
+        self.emit(ins);
     }
 
     /// Implement boolean short circuit evaluation logic.
@@ -2124,13 +2118,9 @@ impl Compiler {
             // Create an optional map with kw-args:
             if !keywords.is_empty() {
                 self.compile_keywords(keywords)?;
-                self.emit(Instruction::CallFunction {
-                    typ: CallType::Ex(true),
-                });
+                self.emit(Instruction::CallFunctionEx { has_kwargs: true });
             } else {
-                self.emit(Instruction::CallFunction {
-                    typ: CallType::Ex(false),
-                });
+                self.emit(Instruction::CallFunctionEx { has_kwargs: false });
             }
         } else {
             // Keyword arguments:
@@ -2151,13 +2141,9 @@ impl Compiler {
                 self.emit_constant(ConstantData::Tuple {
                     elements: kwarg_names,
                 });
-                self.emit(Instruction::CallFunction {
-                    typ: CallType::Keyword(count),
-                });
+                self.emit(Instruction::CallFunctionKeyword { nargs: count });
             } else {
-                self.emit(Instruction::CallFunction {
-                    typ: CallType::Positional(count),
-                });
+                self.emit(Instruction::CallFunctionPositional { nargs: count });
             }
         }
         Ok(())
@@ -2270,10 +2256,7 @@ impl Compiler {
             let start_label = self.new_label();
             let end_label = self.new_label();
             loop_labels.push((start_label, end_label));
-            self.emit(Instruction::SetupLoop {
-                start: start_label,
-                end: end_label,
-            });
+            self.emit(Instruction::SetupLoop { end: end_label });
             self.set_label(start_label);
             self.emit(Instruction::ForIter { target: end_label });
 
@@ -2366,9 +2349,7 @@ impl Compiler {
         self.emit(Instruction::GetIter);
 
         // Call just created <listcomp> function:
-        self.emit(Instruction::CallFunction {
-            typ: CallType::Positional(1),
-        });
+        self.emit(Instruction::CallFunctionPositional { nargs: 1 });
         Ok(())
     }
 
diff --git a/src/error.rs b/src/error.rs
index b5b1d866..f1fa58e5 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -37,6 +37,7 @@ pub enum CompileErrorType {
     InvalidFuturePlacement,
     InvalidFutureFeature(String),
     FunctionImportStar,
+    TooManyStarUnpack,
 }
 
 impl fmt::Display for CompileErrorType {
@@ -70,6 +71,9 @@ impl fmt::Display for CompileErrorType {
             CompileErrorType::FunctionImportStar => {
                 write!(f, "import * only allowed at module level")
             }
+            CompileErrorType::TooManyStarUnpack => {
+                write!(f, "too many expressions in star-unpacking assignment")
+            }
         }
     }
 }

From 74a917a72eea58022dfa4a5f9a6229e8a424a570 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Thu, 10 Dec 2020 13:36:45 -0600
Subject: [PATCH 175/459] Make Continue have the target it's continuing to

---
 src/compile.rs | 52 ++++++++++++++++++++++++++++----------------------
 1 file changed, 29 insertions(+), 23 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index e953ec83..dc0b0530 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -88,7 +88,8 @@ impl CodeInfo {
                 | SetupExcept { handler: l }
                 | SetupWith { end: l }
                 | SetupAsyncWith { end: l }
-                | SetupLoop { end: l } => {
+                | SetupLoop { end: l }
+                | Continue { target: l } => {
                     *l = label_map[l.0].expect("label never set");
                 }
 
@@ -131,7 +132,7 @@ impl Default for CompileOpts {
 
 #[derive(Debug, Clone, Copy)]
 struct CompileContext {
-    in_loop: bool,
+    loop_data: Option<(Label, Label)>,
     in_class: bool,
     func: FunctionContext,
 }
@@ -144,6 +145,9 @@ enum FunctionContext {
 }
 
 impl CompileContext {
+    fn in_loop(self) -> bool {
+        self.loop_data.is_some()
+    }
     fn in_func(self) -> bool {
         self.func != FunctionContext::NoFunction
     }
@@ -236,7 +240,7 @@ impl Compiler {
             current_qualified_path: None,
             done_with_future_stmts: false,
             ctx: CompileContext {
-                in_loop: false,
+                loop_data: None,
                 in_class: false,
                 func: FunctionContext::NoFunction,
             },
@@ -737,19 +741,21 @@ impl Compiler {
                 }
             }
             Break => {
-                if !self.ctx.in_loop {
+                if !self.ctx.in_loop() {
                     return Err(self.error_loc(CompileErrorType::InvalidBreak, statement.location));
                 }
                 self.emit(Instruction::Break);
             }
-            Continue => {
-                if !self.ctx.in_loop {
+            Continue => match self.ctx.loop_data {
+                Some((start, _)) => {
+                    self.emit(Instruction::Continue { target: start });
+                }
+                None => {
                     return Err(
                         self.error_loc(CompileErrorType::InvalidContinue, statement.location)
                     );
                 }
-                self.emit(Instruction::Continue);
-            }
+            },
             Return { value } => {
                 if !self.ctx.in_func() {
                     return Err(self.error_loc(CompileErrorType::InvalidReturn, statement.location));
@@ -1052,7 +1058,7 @@ impl Compiler {
         let prev_ctx = self.ctx;
 
         self.ctx = CompileContext {
-            in_loop: false,
+            loop_data: None,
             in_class: prev_ctx.in_class,
             func: if is_async {
                 FunctionContext::AsyncFunction
@@ -1258,7 +1264,7 @@ impl Compiler {
         self.ctx = CompileContext {
             func: FunctionContext::NoFunction,
             in_class: true,
-            in_loop: false,
+            loop_data: None,
         };
 
         let qualified_name = self.create_qualified_name(name, "");
@@ -1399,10 +1405,10 @@ impl Compiler {
 
         self.compile_jump_if(test, false, else_label)?;
 
-        let was_in_loop = self.ctx.in_loop;
-        self.ctx.in_loop = true;
+        let was_in_loop = self.ctx.loop_data;
+        self.ctx.loop_data = Some((start_label, end_label));
         self.compile_statements(body)?;
-        self.ctx.in_loop = was_in_loop;
+        self.ctx.loop_data = was_in_loop;
         self.emit(Instruction::Jump {
             target: start_label,
         });
@@ -1428,6 +1434,8 @@ impl Compiler {
         let else_label = self.new_label();
         let end_label = self.new_label();
 
+        self.emit(Instruction::SetupLoop { end: end_label });
+
         // The thing iterated:
         self.compile_expression(iter)?;
 
@@ -1437,7 +1445,6 @@ impl Compiler {
 
             self.emit(Instruction::GetAIter);
 
-            self.emit(Instruction::SetupLoop { end: end_label });
             self.set_label(start_label);
             self.emit(Instruction::SetupExcept {
                 handler: check_asynciter_label,
@@ -1459,26 +1466,25 @@ impl Compiler {
             self.emit(Instruction::JumpIfTrue { target: else_label });
             self.emit(Instruction::Raise { argc: 0 });
 
-            let was_in_loop = self.ctx.in_loop;
-            self.ctx.in_loop = true;
+            let was_in_loop = self.ctx.loop_data;
+            self.ctx.loop_data = Some((start_label, end_label));
             self.set_label(body_label);
             self.compile_statements(body)?;
-            self.ctx.in_loop = was_in_loop;
+            self.ctx.loop_data = was_in_loop;
         } else {
             // Retrieve Iterator
             self.emit(Instruction::GetIter);
 
-            self.emit(Instruction::SetupLoop { end: end_label });
             self.set_label(start_label);
             self.emit(Instruction::ForIter { target: else_label });
 
             // Start of loop iteration, set targets:
             self.compile_store(target)?;
 
-            let was_in_loop = self.ctx.in_loop;
-            self.ctx.in_loop = true;
+            let was_in_loop = self.ctx.loop_data;
+            self.ctx.loop_data = Some((start_label, end_label));
             self.compile_statements(body)?;
-            self.ctx.in_loop = was_in_loop;
+            self.ctx.loop_data = was_in_loop;
         }
 
         self.emit(Instruction::Jump {
@@ -2009,7 +2015,7 @@ impl Compiler {
             Lambda { args, body } => {
                 let prev_ctx = self.ctx;
                 self.ctx = CompileContext {
-                    in_loop: false,
+                    loop_data: Option::None,
                     in_class: prev_ctx.in_class,
                     func: FunctionContext::Function,
                 };
@@ -2182,7 +2188,7 @@ impl Compiler {
         let prev_ctx = self.ctx;
 
         self.ctx = CompileContext {
-            in_loop: false,
+            loop_data: None,
             in_class: prev_ctx.in_class,
             func: FunctionContext::Function,
         };

From 86aa00c473f4d3c7bdbbe44e08b1da8ad8a2b64e Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Thu, 10 Dec 2020 18:44:12 -0600
Subject: [PATCH 176/459] Add label_arg{,_mut} methods

---
 src/compile.rs | 31 ++++++++-----------------------
 1 file changed, 8 insertions(+), 23 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index dc0b0530..b30c7e61 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -26,7 +26,7 @@ struct CodeInfo {
     varname_cache: IndexSet<String>,
     cellvar_cache: IndexSet<String>,
     freevar_cache: IndexSet<String>,
-    label_map: Vec<Option<Label>>,
+    label_map: Vec<Label>,
 }
 impl CodeInfo {
     fn finalize_code(self) -> CodeObject {
@@ -73,27 +73,12 @@ impl CodeInfo {
         }
 
         for instruction in &mut *code.instructions {
-            use Instruction::*;
             // this is a little bit hacky, as until now the data stored inside Labels in
             // Instructions is just bookkeeping, but I think it's the best way to do this
-            // XXX: any new instruction that uses a label has to be added here
-            match instruction {
-                Jump { target: l }
-                | JumpIfTrue { target: l }
-                | JumpIfFalse { target: l }
-                | JumpIfTrueOrPop { target: l }
-                | JumpIfFalseOrPop { target: l }
-                | ForIter { target: l }
-                | SetupFinally { handler: l }
-                | SetupExcept { handler: l }
-                | SetupWith { end: l }
-                | SetupAsyncWith { end: l }
-                | SetupLoop { end: l }
-                | Continue { target: l } => {
-                    *l = label_map[l.0].expect("label never set");
-                }
-
-                _ => {}
+            if let Some(l) = instruction.label_arg_mut() {
+                let real_label = label_map[l.0];
+                debug_assert!(real_label.0 != usize::MAX, "label wasn't set");
+                *l = real_label;
             }
         }
         code
@@ -2446,7 +2431,7 @@ impl Compiler {
     fn new_label(&mut self) -> Label {
         let label_map = &mut self.current_codeinfo().label_map;
         let label = Label(label_map.len());
-        label_map.push(None);
+        label_map.push(Label(usize::MAX));
         label
     }
 
@@ -2458,9 +2443,9 @@ impl Compiler {
             ..
         } = self.current_codeinfo();
         let actual_label = Label(instructions.len());
-        let prev_val = std::mem::replace(&mut label_map[label.0], Some(actual_label));
+        let prev_val = std::mem::replace(&mut label_map[label.0], actual_label);
         debug_assert!(
-            prev_val.map_or(true, |x| x == actual_label),
+            prev_val.0 == usize::MAX || prev_val == actual_label,
             "double-set a label"
         );
     }

From 1575382577d42699bbc0d28195575715c4f9f69a Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 4 Dec 2020 13:38:18 -0600
Subject: [PATCH 177/459] Add flags to the MakeFunction instruction

---
 src/compile.rs | 99 +++++++++++++++++++++++++++++---------------------
 src/lib.rs     |  3 +-
 2 files changed, 59 insertions(+), 43 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index b30c7e61..270d0449 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -827,7 +827,11 @@ impl Compiler {
         Ok(())
     }
 
-    fn enter_function(&mut self, name: &str, args: &ast::Parameters) -> CompileResult<()> {
+    fn enter_function(
+        &mut self,
+        name: &str,
+        args: &ast::Parameters,
+    ) -> CompileResult<bytecode::MakeFunctionFlags> {
         let have_defaults = !args.defaults.is_empty();
         if have_defaults {
             // Construct a tuple:
@@ -859,17 +863,17 @@ impl Compiler {
             });
         }
 
-        let mut flags = bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED;
+        let mut funcflags = bytecode::MakeFunctionFlags::empty();
         if have_defaults {
-            flags |= bytecode::CodeFlags::HAS_DEFAULTS;
+            funcflags |= bytecode::MakeFunctionFlags::DEFAULTS;
         }
         if num_kw_only_defaults > 0 {
-            flags |= bytecode::CodeFlags::HAS_KW_ONLY_DEFAULTS;
+            funcflags |= bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS;
         }
 
         let line_number = self.get_source_line_number();
         self.push_output(CodeObject::new(
-            flags,
+            bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED,
             args.posonlyargs_count,
             args.args.len(),
             args.kwonlyargs.len(),
@@ -896,7 +900,7 @@ impl Compiler {
         compile_varargs(&args.vararg, bytecode::CodeFlags::HAS_VARARGS);
         compile_varargs(&args.kwarg, bytecode::CodeFlags::HAS_VARKEYWORDS);
 
-        Ok(())
+        Ok(funcflags)
     }
 
     fn prepare_decorators(&mut self, decorator_list: &[ast::Expression]) -> CompileResult<()> {
@@ -1037,7 +1041,7 @@ impl Compiler {
         // Create bytecode for this function:
 
         self.prepare_decorators(decorator_list)?;
-        self.enter_function(name, args)?;
+        let mut funcflags = self.enter_function(name, args)?;
 
         // remember to restore self.ctx.in_loop to the original after the function is compiled
         let prev_ctx = self.ctx;
@@ -1113,7 +1117,7 @@ impl Compiler {
         }
 
         if num_annotations > 0 {
-            code.flags |= bytecode::CodeFlags::HAS_ANNOTATIONS;
+            funcflags |= bytecode::MakeFunctionFlags::ANNOTATIONS;
             self.emit(Instruction::BuildMap {
                 size: num_annotations,
                 unpack: false,
@@ -1125,7 +1129,9 @@ impl Compiler {
             code.flags |= bytecode::CodeFlags::IS_COROUTINE;
         }
 
-        self.build_closure(&code);
+        if self.build_closure(&code) {
+            funcflags |= bytecode::MakeFunctionFlags::CLOSURE;
+        }
 
         self.emit_constant(ConstantData::Code {
             code: Box::new(code),
@@ -1135,7 +1141,7 @@ impl Compiler {
         });
 
         // Turn code object into function object:
-        self.emit(Instruction::MakeFunction);
+        self.emit(Instruction::MakeFunction(funcflags));
 
         self.emit(Instruction::Duplicate);
         self.load_docstring(doc_str);
@@ -1150,32 +1156,34 @@ impl Compiler {
         Ok(())
     }
 
-    fn build_closure(&mut self, code: &CodeObject) {
-        if !code.freevars.is_empty() {
-            for var in &*code.freevars {
-                let table = self.symbol_table_stack.last().unwrap();
-                let symbol = table.lookup(var).unwrap();
-                let parent_code = self.code_stack.last().unwrap();
-                let vars = match symbol.scope {
-                    SymbolScope::Free => &parent_code.freevar_cache,
-                    SymbolScope::Cell => &parent_code.cellvar_cache,
-                    _ if symbol.is_free_class => &parent_code.freevar_cache,
-                    x => unreachable!(
-                        "var {} in a {:?} should be free or cell but it's {:?}",
-                        var, table.typ, x
-                    ),
-                };
-                let mut idx = vars.get_index_of(var).unwrap();
-                if let SymbolScope::Free = symbol.scope {
-                    idx += parent_code.cellvar_cache.len();
-                }
-                self.emit(Instruction::LoadClosure(idx))
+    fn build_closure(&mut self, code: &CodeObject) -> bool {
+        if code.freevars.is_empty() {
+            return false;
+        }
+        for var in &*code.freevars {
+            let table = self.symbol_table_stack.last().unwrap();
+            let symbol = table.lookup(var).unwrap();
+            let parent_code = self.code_stack.last().unwrap();
+            let vars = match symbol.scope {
+                SymbolScope::Free => &parent_code.freevar_cache,
+                SymbolScope::Cell => &parent_code.cellvar_cache,
+                _ if symbol.is_free_class => &parent_code.freevar_cache,
+                x => unreachable!(
+                    "var {} in a {:?} should be free or cell but it's {:?}",
+                    var, table.typ, x
+                ),
+            };
+            let mut idx = vars.get_index_of(var).unwrap();
+            if let SymbolScope::Free = symbol.scope {
+                idx += parent_code.cellvar_cache.len();
             }
-            self.emit(Instruction::BuildTuple {
-                size: code.freevars.len(),
-                unpack: false,
-            })
+            self.emit(Instruction::LoadClosure(idx))
         }
+        self.emit(Instruction::BuildTuple {
+            size: code.freevars.len(),
+            unpack: false,
+        });
+        true
     }
 
     fn find_ann(&self, body: &[ast::Statement]) -> bool {
@@ -1312,7 +1320,11 @@ impl Compiler {
         self.current_qualified_path = old_qualified_path;
         self.ctx = prev_ctx;
 
-        self.build_closure(&code);
+        let mut funcflags = bytecode::MakeFunctionFlags::empty();
+
+        if self.build_closure(&code) {
+            funcflags |= bytecode::MakeFunctionFlags::CLOSURE;
+        }
 
         self.emit_constant(ConstantData::Code {
             code: Box::new(code),
@@ -1322,7 +1334,7 @@ impl Compiler {
         });
 
         // Turn code object into function object:
-        self.emit(Instruction::MakeFunction);
+        self.emit(Instruction::MakeFunction(funcflags));
 
         self.emit_constant(ConstantData::Str {
             value: qualified_name,
@@ -2006,17 +2018,19 @@ impl Compiler {
                 };
 
                 let name = "<lambda>".to_owned();
-                self.enter_function(&name, args)?;
+                let mut funcflags = self.enter_function(&name, args)?;
                 self.compile_expression(body)?;
                 self.emit(Instruction::ReturnValue);
                 let code = self.pop_code_object();
-                self.build_closure(&code);
+                if self.build_closure(&code) {
+                    funcflags |= bytecode::MakeFunctionFlags::CLOSURE;
+                }
                 self.emit_constant(ConstantData::Code {
                     code: Box::new(code),
                 });
                 self.emit_constant(ConstantData::Str { value: name });
                 // Turn code object into function object:
-                self.emit(Instruction::MakeFunction);
+                self.emit(Instruction::MakeFunction(funcflags));
 
                 self.ctx = prev_ctx;
             }
@@ -2320,7 +2334,10 @@ impl Compiler {
 
         self.ctx = prev_ctx;
 
-        self.build_closure(&code);
+        let mut funcflags = bytecode::MakeFunctionFlags::empty();
+        if self.build_closure(&code) {
+            funcflags |= bytecode::MakeFunctionFlags::CLOSURE;
+        }
 
         // List comprehension code:
         self.emit_constant(ConstantData::Code {
@@ -2331,7 +2348,7 @@ impl Compiler {
         self.emit_constant(ConstantData::Str { value: name });
 
         // Turn code object into function object:
-        self.emit(Instruction::MakeFunction);
+        self.emit(Instruction::MakeFunction(funcflags));
 
         // Evaluate iterated item:
         self.compile_expression(&generators[0].iter)?;
diff --git a/src/lib.rs b/src/lib.rs
index 415ff613..ab299864 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,5 +1,4 @@
-//! Compile a Python AST or source code into bytecode consumable by RustPython or
-//! (eventually) CPython.
+//! Compile a Python AST or source code into bytecode consumable by RustPython.
 #![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/master/logo.png")]
 #![doc(html_root_url = "https://docs.rs/rustpython-compiler/")]
 

From b9875ff9bf148794e87c3b68a2a22e08ac3780c6 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 14 Dec 2020 15:55:47 -0600
Subject: [PATCH 178/459] Make Instruction only 8 bytes

---
 src/compile.rs | 112 +++++++++++++++++++++++++++----------------------
 1 file changed, 61 insertions(+), 51 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 270d0449..0847cab4 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -76,8 +76,8 @@ impl CodeInfo {
             // this is a little bit hacky, as until now the data stored inside Labels in
             // Instructions is just bookkeeping, but I think it's the best way to do this
             if let Some(l) = instruction.label_arg_mut() {
-                let real_label = label_map[l.0];
-                debug_assert!(real_label.0 != usize::MAX, "label wasn't set");
+                let real_label = label_map[l.0 as usize];
+                debug_assert!(real_label.0 != u32::MAX, "label wasn't set");
                 *l = real_label;
             }
         }
@@ -303,7 +303,7 @@ impl Compiler {
         let cache = cache(self.current_codeinfo());
         cache
             .get_index_of(name)
-            .unwrap_or_else(|| cache.insert_full(name.to_owned()).0)
+            .unwrap_or_else(|| cache.insert_full(name.to_owned()).0) as u32
     }
 
     fn compile_program(
@@ -470,7 +470,7 @@ impl Compiler {
                 NameUsage::Delete => Instruction::DeleteLocal,
             },
         };
-        self.emit(op(idx));
+        self.emit(op(idx as u32));
     }
 
     fn compile_statement(&mut self, statement: &ast::Statement) -> CompileResult<()> {
@@ -665,23 +665,22 @@ impl Compiler {
                 body,
                 orelse,
             } => self.compile_for(target, iter, body, orelse, *is_async)?,
-            Raise { exception, cause } => match exception {
-                Some(value) => {
-                    self.compile_expression(value)?;
-                    match cause {
-                        Some(cause) => {
-                            self.compile_expression(cause)?;
-                            self.emit(Instruction::Raise { argc: 2 });
-                        }
-                        None => {
-                            self.emit(Instruction::Raise { argc: 1 });
+            Raise { exception, cause } => {
+                let kind = match exception {
+                    Some(value) => {
+                        self.compile_expression(value)?;
+                        match cause {
+                            Some(cause) => {
+                                self.compile_expression(cause)?;
+                                bytecode::RaiseKind::RaiseCause
+                            }
+                            None => bytecode::RaiseKind::Raise,
                         }
                     }
-                }
-                None => {
-                    self.emit(Instruction::Raise { argc: 0 });
-                }
-            },
+                    None => bytecode::RaiseKind::Reraise,
+                };
+                self.emit(Instruction::Raise { kind });
+            }
             Try {
                 body,
                 handlers,
@@ -721,7 +720,9 @@ impl Compiler {
                             self.emit(Instruction::CallFunctionPositional { nargs: 0 });
                         }
                     }
-                    self.emit(Instruction::Raise { argc: 1 });
+                    self.emit(Instruction::Raise {
+                        kind: bytecode::RaiseKind::Raise,
+                    });
                     self.set_label(end_label);
                 }
             }
@@ -835,7 +836,7 @@ impl Compiler {
         let have_defaults = !args.defaults.is_empty();
         if have_defaults {
             // Construct a tuple:
-            let size = args.defaults.len();
+            let size = args.defaults.len() as u32;
             for element in &args.defaults {
                 self.compile_expression(element)?;
             }
@@ -1003,7 +1004,9 @@ impl Compiler {
         self.set_label(handler_label);
         // If code flows here, we have an unhandled exception,
         // raise the exception again!
-        self.emit(Instruction::Raise { argc: 0 });
+        self.emit(Instruction::Raise {
+            kind: bytecode::RaiseKind::Reraise,
+        });
 
         // We successfully ran the try block:
         // else:
@@ -1177,10 +1180,10 @@ impl Compiler {
             if let SymbolScope::Free = symbol.scope {
                 idx += parent_code.cellvar_cache.len();
             }
-            self.emit(Instruction::LoadClosure(idx))
+            self.emit(Instruction::LoadClosure(idx as u32))
         }
         self.emit(Instruction::BuildTuple {
-            size: code.freevars.len(),
+            size: code.freevars.len() as u32,
             unpack: false,
         });
         true
@@ -1305,7 +1308,7 @@ impl Compiler {
             .position(|var| *var == "__class__");
 
         if let Some(classcell_idx) = classcell_idx {
-            self.emit(Instruction::LoadClosure(classcell_idx));
+            self.emit(Instruction::LoadClosure(classcell_idx as u32));
             self.emit(Instruction::Duplicate);
             let classcell = self.name("__classcell__");
             self.emit(Instruction::StoreLocal(classcell));
@@ -1362,11 +1365,11 @@ impl Compiler {
                 elements: kwarg_names,
             });
             self.emit(Instruction::CallFunctionKeyword {
-                nargs: 2 + keywords.len() + bases.len(),
+                nargs: (2 + keywords.len() + bases.len()) as u32,
             });
         } else {
             self.emit(Instruction::CallFunctionPositional {
-                nargs: 2 + bases.len(),
+                nargs: (2 + bases.len()) as u32,
             });
         }
 
@@ -1461,7 +1464,9 @@ impl Compiler {
                 op: bytecode::ComparisonOperator::ExceptionMatch,
             });
             self.emit(Instruction::JumpIfTrue { target: else_label });
-            self.emit(Instruction::Raise { argc: 0 });
+            self.emit(Instruction::Raise {
+                kind: bytecode::RaiseKind::Reraise,
+            });
 
             let was_in_loop = self.ctx.loop_data;
             self.ctx.loop_data = Some((start_label, end_label));
@@ -1649,7 +1654,7 @@ impl Compiler {
 
                 if !seen_star {
                     self.emit(Instruction::UnpackSequence {
-                        size: elements.len(),
+                        size: elements.len() as u32,
                     });
                 }
 
@@ -1917,7 +1922,7 @@ impl Compiler {
                 self.emit_constant(const_value);
             }
             List { elements } => {
-                let size = elements.len();
+                let size = elements.len() as u32;
                 let must_unpack = self.gather_elements(elements)?;
                 self.emit(Instruction::BuildList {
                     size,
@@ -1925,7 +1930,7 @@ impl Compiler {
                 });
             }
             Tuple { elements } => {
-                let size = elements.len();
+                let size = elements.len() as u32;
                 let must_unpack = self.gather_elements(elements)?;
                 self.emit(Instruction::BuildTuple {
                     size,
@@ -1933,7 +1938,7 @@ impl Compiler {
                 });
             }
             Set { elements } => {
-                let size = elements.len();
+                let size = elements.len() as u32;
                 let must_unpack = self.gather_elements(elements)?;
                 self.emit(Instruction::BuildSet {
                     size,
@@ -1944,11 +1949,11 @@ impl Compiler {
                 self.compile_dict(elements)?;
             }
             Slice { elements } => {
-                let size = elements.len();
+                let step = elements.len() >= 3;
                 for element in elements {
                     self.compile_expression(element)?;
                 }
-                self.emit(Instruction::BuildSlice { size });
+                self.emit(Instruction::BuildSlice { step });
             }
             Yield { value } => {
                 if !self.ctx.in_func() {
@@ -2107,7 +2112,7 @@ impl Compiler {
         keywords: &[ast::Keyword],
     ) -> CompileResult<()> {
         self.compile_expression(function)?;
-        let count = args.len() + keywords.len();
+        let count = (args.len() + keywords.len()) as u32;
 
         // Normal arguments:
         let must_unpack = self.gather_elements(args)?;
@@ -2116,7 +2121,7 @@ impl Compiler {
         if must_unpack || has_double_star {
             // Create a tuple with positional args:
             self.emit(Instruction::BuildTuple {
-                size: args.len(),
+                size: args.len() as u32,
                 unpack: must_unpack,
             });
 
@@ -2295,13 +2300,13 @@ impl Compiler {
             ast::ComprehensionKind::List { element } => {
                 compile_element(element)?;
                 self.emit(Instruction::ListAppend {
-                    i: 1 + generators.len(),
+                    i: (1 + generators.len()) as u32,
                 });
             }
             ast::ComprehensionKind::Set { element } => {
                 compile_element(element)?;
                 self.emit(Instruction::SetAdd {
-                    i: 1 + generators.len(),
+                    i: (1 + generators.len()) as u32,
                 });
             }
             ast::ComprehensionKind::Dict { key, value } => {
@@ -2310,7 +2315,7 @@ impl Compiler {
                 self.compile_expression(value)?;
 
                 self.emit(Instruction::MapAddRev {
-                    i: 1 + generators.len(),
+                    i: (1 + generators.len()) as u32,
                 });
             }
         }
@@ -2370,7 +2375,9 @@ impl Compiler {
                     for value in values {
                         self.compile_string(value)?;
                     }
-                    self.emit(Instruction::BuildString { size: values.len() })
+                    self.emit(Instruction::BuildString {
+                        size: values.len() as u32,
+                    })
                 }
                 ast::StringGroup::Constant { value } => {
                     self.emit_constant(ConstantData::Str {
@@ -2390,7 +2397,7 @@ impl Compiler {
                     };
                     self.compile_expression(value)?;
                     self.emit(Instruction::FormatValue {
-                        conversion: conversion.map(compile_conversion_flag),
+                        conversion: compile_conversion_flag(*conversion),
                     });
                 }
             }
@@ -2431,7 +2438,7 @@ impl Compiler {
 
     fn emit_constant(&mut self, constant: ConstantData) {
         let info = self.current_codeinfo();
-        let idx = info.constants.len();
+        let idx = info.constants.len() as u32;
         info.constants.push(constant);
         self.emit(Instruction::LoadConst { idx })
     }
@@ -2447,8 +2454,8 @@ impl Compiler {
     // Generate a new label
     fn new_label(&mut self) -> Label {
         let label_map = &mut self.current_codeinfo().label_map;
-        let label = Label(label_map.len());
-        label_map.push(Label(usize::MAX));
+        let label = Label(label_map.len() as u32);
+        label_map.push(Label(u32::MAX));
         label
     }
 
@@ -2459,10 +2466,10 @@ impl Compiler {
             label_map,
             ..
         } = self.current_codeinfo();
-        let actual_label = Label(instructions.len());
-        let prev_val = std::mem::replace(&mut label_map[label.0], actual_label);
+        let actual_label = Label(instructions.len() as u32);
+        let prev_val = std::mem::replace(&mut label_map[label.0 as usize], actual_label);
         debug_assert!(
-            prev_val.0 == usize::MAX || prev_val == actual_label,
+            prev_val.0 == u32::MAX || prev_val == actual_label,
             "double-set a label"
         );
     }
@@ -2526,11 +2533,14 @@ fn compile_location(location: &ast::Location) -> bytecode::Location {
     bytecode::Location::new(location.row(), location.column())
 }
 
-fn compile_conversion_flag(conversion_flag: ast::ConversionFlag) -> bytecode::ConversionFlag {
+fn compile_conversion_flag(
+    conversion_flag: Option<ast::ConversionFlag>,
+) -> bytecode::ConversionFlag {
     match conversion_flag {
-        ast::ConversionFlag::Ascii => bytecode::ConversionFlag::Ascii,
-        ast::ConversionFlag::Repr => bytecode::ConversionFlag::Repr,
-        ast::ConversionFlag::Str => bytecode::ConversionFlag::Str,
+        None => bytecode::ConversionFlag::None,
+        Some(ast::ConversionFlag::Ascii) => bytecode::ConversionFlag::Ascii,
+        Some(ast::ConversionFlag::Repr) => bytecode::ConversionFlag::Repr,
+        Some(ast::ConversionFlag::Str) => bytecode::ConversionFlag::Str,
     }
 }
 

From db041c0f6c2d2982379ded13499fe164c1d10dfd Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 19 Dec 2020 14:42:52 -0600
Subject: [PATCH 179/459] Update other stuff to use the root of
 rustpython_bytecode

---
 porcelain/src/lib.rs | 2 +-
 src/compile.rs       | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/porcelain/src/lib.rs b/porcelain/src/lib.rs
index bdf93ee0..9996c042 100644
--- a/porcelain/src/lib.rs
+++ b/porcelain/src/lib.rs
@@ -1,4 +1,4 @@
-use rustpython_bytecode::bytecode::CodeObject;
+use rustpython_bytecode::CodeObject;
 use rustpython_compiler_core::{compile, symboltable};
 use rustpython_parser::{ast::Location, parser};
 use std::fmt;
diff --git a/src/compile.rs b/src/compile.rs
index 0847cab4..dc4cec7e 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -13,7 +13,7 @@ use itertools::Itertools;
 use num_complex::Complex64;
 use num_traits::ToPrimitive;
 use rustpython_ast as ast;
-use rustpython_bytecode::bytecode::{self, CodeObject, ConstantData, Instruction, Label};
+use rustpython_bytecode::{self as bytecode, CodeObject, ConstantData, Instruction, Label};
 
 type CompileResult<T> = Result<T, CompileError>;
 
@@ -2548,7 +2548,7 @@ fn compile_conversion_flag(
 mod tests {
     use super::{CompileOpts, Compiler};
     use crate::symboltable::make_symbol_table;
-    use rustpython_bytecode::bytecode::CodeObject;
+    use rustpython_bytecode::CodeObject;
     use rustpython_parser::parser;
 
     fn compile_exec(source: &str) -> CodeObject {

From 596e338c512d637b0b29e32490bfab0f7cef4a82 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 16 Dec 2020 15:29:56 -0600
Subject: [PATCH 180/459] Have a block-based IR for compiling, calculate max
 stack size

---
 src/compile.rs | 463 +++++++++++++++++++++++--------------------------
 src/ir.rs      | 224 ++++++++++++++++++++++++
 src/lib.rs     |   1 +
 3 files changed, 440 insertions(+), 248 deletions(-)
 create mode 100644 src/ir.rs

diff --git a/src/compile.rs b/src/compile.rs
index dc4cec7e..739be8ed 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -6,6 +6,7 @@
 //!   https://github.com/micropython/micropython/blob/master/py/compile.c
 
 use crate::error::{CompileError, CompileErrorType};
+use crate::ir::{self, CodeInfo};
 pub use crate::mode::Mode;
 use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolScope, SymbolTable};
 use indexmap::IndexSet;
@@ -13,78 +14,10 @@ use itertools::Itertools;
 use num_complex::Complex64;
 use num_traits::ToPrimitive;
 use rustpython_ast as ast;
-use rustpython_bytecode::{self as bytecode, CodeObject, ConstantData, Instruction, Label};
+use rustpython_bytecode::{self as bytecode, CodeObject, ConstantData, Instruction};
 
 type CompileResult<T> = Result<T, CompileError>;
 
-struct CodeInfo {
-    code: CodeObject,
-    instructions: Vec<Instruction>,
-    locations: Vec<bytecode::Location>,
-    constants: Vec<ConstantData>,
-    name_cache: IndexSet<String>,
-    varname_cache: IndexSet<String>,
-    cellvar_cache: IndexSet<String>,
-    freevar_cache: IndexSet<String>,
-    label_map: Vec<Label>,
-}
-impl CodeInfo {
-    fn finalize_code(self) -> CodeObject {
-        let CodeInfo {
-            mut code,
-            instructions,
-            locations,
-            constants,
-            name_cache,
-            varname_cache,
-            cellvar_cache,
-            freevar_cache,
-            label_map,
-        } = self;
-
-        code.instructions = instructions.into();
-        code.locations = locations.into();
-        code.constants = constants.into();
-        code.names = name_cache.into_iter().collect();
-        code.varnames = varname_cache.into_iter().collect();
-        code.cellvars = cellvar_cache.into_iter().collect();
-        code.freevars = freevar_cache.into_iter().collect();
-
-        if !code.cellvars.is_empty() {
-            let total_args = code.arg_count
-                + code.kwonlyarg_count
-                + code.flags.contains(bytecode::CodeFlags::HAS_VARARGS) as usize
-                + code.flags.contains(bytecode::CodeFlags::HAS_VARKEYWORDS) as usize;
-            let all_args = &code.varnames[..total_args];
-            let mut found_cellarg = false;
-            let cell2arg = code
-                .cellvars
-                .iter()
-                .map(|var| {
-                    all_args.iter().position(|arg| var == arg).map_or(-1, |i| {
-                        found_cellarg = true;
-                        i as isize
-                    })
-                })
-                .collect::<Box<[_]>>();
-            if found_cellarg {
-                code.cell2arg = Some(cell2arg);
-            }
-        }
-
-        for instruction in &mut *code.instructions {
-            // this is a little bit hacky, as until now the data stored inside Labels in
-            // Instructions is just bookkeeping, but I think it's the best way to do this
-            if let Some(l) = instruction.label_arg_mut() {
-                let real_label = label_map[l.0 as usize];
-                debug_assert!(real_label.0 != u32::MAX, "label wasn't set");
-                *l = real_label;
-            }
-        }
-        code
-    }
-}
-
 enum NameUsage {
     Load,
     Store,
@@ -117,7 +50,7 @@ impl Default for CompileOpts {
 
 #[derive(Debug, Clone, Copy)]
 struct CompileContext {
-    loop_data: Option<(Label, Label)>,
+    loop_data: Option<(ir::BlockIdx, ir::BlockIdx)>,
     in_class: bool,
     func: FunctionContext,
 }
@@ -130,9 +63,6 @@ enum FunctionContext {
 }
 
 impl CompileContext {
-    fn in_loop(self) -> bool {
-        self.loop_data.is_some()
-    }
     fn in_func(self) -> bool {
         self.func != FunctionContext::NoFunction
     }
@@ -199,23 +129,21 @@ pub fn compile_program_single(
 impl Compiler {
     fn new(opts: CompileOpts, source_path: String, code_name: String) -> Self {
         let module_code = CodeInfo {
-            code: CodeObject::new(
-                bytecode::CodeFlags::NEW_LOCALS,
-                0,
-                0,
-                0,
-                source_path.clone(),
-                0,
-                code_name,
-            ),
-            instructions: Vec::new(),
-            locations: Vec::new(),
+            flags: bytecode::CodeFlags::NEW_LOCALS,
+            posonlyarg_count: 0,
+            arg_count: 0,
+            kwonlyarg_count: 0,
+            source_path: source_path.clone(),
+            first_line_number: 0,
+            obj_name: code_name,
+
+            blocks: vec![ir::Block::default()],
+            block_order: vec![bytecode::Label(0)],
             constants: Vec::new(),
             name_cache: IndexSet::new(),
             varname_cache: IndexSet::new(),
             cellvar_cache: IndexSet::new(),
             freevar_cache: IndexSet::new(),
-            label_map: Vec::new(),
         };
         Compiler {
             code_stack: vec![module_code],
@@ -244,7 +172,16 @@ impl Compiler {
         }
     }
 
-    fn push_output(&mut self, code: CodeObject) {
+    fn push_output(
+        &mut self,
+        flags: bytecode::CodeFlags,
+        posonlyarg_count: usize,
+        arg_count: usize,
+        kwonlyarg_count: usize,
+        source_path: String,
+        first_line_number: usize,
+        obj_name: String,
+    ) {
         let table = self
             .symbol_table_stack
             .last_mut()
@@ -268,15 +205,21 @@ impl Compiler {
         self.symbol_table_stack.push(table);
 
         let info = CodeInfo {
-            code,
-            instructions: Vec::new(),
-            locations: Vec::new(),
+            flags,
+            posonlyarg_count,
+            arg_count,
+            kwonlyarg_count,
+            source_path,
+            first_line_number,
+            obj_name,
+
+            blocks: vec![ir::Block::default()],
+            block_order: vec![bytecode::Label(0)],
             constants: Vec::new(),
             name_cache: IndexSet::new(),
             varname_cache: IndexSet::new(),
             cellvar_cache,
             freevar_cache,
-            label_map: Vec::new(),
         };
         self.code_stack.push(info);
     }
@@ -284,7 +227,10 @@ impl Compiler {
     fn pop_code_object(&mut self) -> CodeObject {
         let table = self.symbol_table_stack.pop().unwrap();
         assert!(table.sub_tables.is_empty());
-        self.code_stack.pop().unwrap().finalize_code()
+        self.code_stack
+            .pop()
+            .unwrap()
+            .finalize_code(self.opts.optimize)
     }
 
     // could take impl Into<Cow<str>>, but everything is borrowed from ast structs; we never
@@ -581,27 +527,28 @@ impl Compiler {
                 // Handled during symbol table construction.
             }
             If { test, body, orelse } => {
-                let end_label = self.new_label();
+                let after_block = self.new_block();
                 match orelse {
                     None => {
                         // Only if:
-                        self.compile_jump_if(test, false, end_label)?;
+                        self.compile_jump_if(test, false, after_block)?;
                         self.compile_statements(body)?;
-                        self.set_label(end_label);
                     }
                     Some(statements) => {
                         // if - else:
-                        let else_label = self.new_label();
-                        self.compile_jump_if(test, false, else_label)?;
+                        let else_block = self.new_block();
+                        self.compile_jump_if(test, false, else_block)?;
                         self.compile_statements(body)?;
-                        self.emit(Instruction::Jump { target: end_label });
+                        self.emit(Instruction::Jump {
+                            target: after_block,
+                        });
 
                         // else:
-                        self.set_label(else_label);
+                        self.switch_to_block(else_block);
                         self.compile_statements(statements)?;
                     }
                 }
-                self.set_label(end_label);
+                self.switch_to_block(after_block);
             }
             While { test, body, orelse } => self.compile_while(test, body, orelse)?,
             With {
@@ -611,10 +558,10 @@ impl Compiler {
             } => {
                 let is_async = *is_async;
 
-                let end_labels = items
+                let end_blocks = items
                     .iter()
                     .map(|item| {
-                        let end_label = self.new_label();
+                        let end_block = self.new_block();
                         self.compile_expression(&item.context_expr)?;
 
                         if is_async {
@@ -622,9 +569,9 @@ impl Compiler {
                             self.emit(Instruction::GetAwaitable);
                             self.emit_constant(ConstantData::None);
                             self.emit(Instruction::YieldFrom);
-                            self.emit(Instruction::SetupAsyncWith { end: end_label });
+                            self.emit(Instruction::SetupAsyncWith { end: end_block });
                         } else {
-                            self.emit(Instruction::SetupWith { end: end_label });
+                            self.emit(Instruction::SetupWith { end: end_block });
                         }
 
                         match &item.optional_vars {
@@ -635,7 +582,7 @@ impl Compiler {
                                 self.emit(Instruction::Pop);
                             }
                         }
-                        Ok(end_label)
+                        Ok(end_block)
                     })
                     .collect::<CompileResult<Vec<_>>>()?;
 
@@ -643,10 +590,11 @@ impl Compiler {
 
                 // sort of "stack up" the layers of with blocks:
                 // with a, b: body -> start_with(a) start_with(b) body() end_with(b) end_with(a)
-                for end_label in end_labels.into_iter().rev() {
+                for end_block in end_blocks.into_iter().rev() {
                     self.emit(Instruction::PopBlock);
                     self.emit(Instruction::EnterFinally);
-                    self.set_label(end_label);
+
+                    self.switch_to_block(end_block);
                     self.emit(Instruction::WithCleanupStart);
 
                     if is_async {
@@ -707,8 +655,9 @@ impl Compiler {
             Assert { test, msg } => {
                 // if some flag, ignore all assert statements!
                 if self.opts.optimize == 0 {
-                    let end_label = self.new_label();
-                    self.compile_jump_if(test, true, end_label)?;
+                    let after_block = self.new_block();
+                    self.compile_jump_if(test, true, after_block)?;
+
                     let assertion_error = self.name("AssertionError");
                     self.emit(Instruction::LoadGlobal(assertion_error));
                     match msg {
@@ -723,15 +672,18 @@ impl Compiler {
                     self.emit(Instruction::Raise {
                         kind: bytecode::RaiseKind::Raise,
                     });
-                    self.set_label(end_label);
+
+                    self.switch_to_block(after_block);
                 }
             }
-            Break => {
-                if !self.ctx.in_loop() {
-                    return Err(self.error_loc(CompileErrorType::InvalidBreak, statement.location));
+            Break => match self.ctx.loop_data {
+                Some((_, end)) => {
+                    self.emit(Instruction::Break { target: end });
                 }
-                self.emit(Instruction::Break);
-            }
+                None => {
+                    return Err(self.error_loc(CompileErrorType::InvalidBreak, statement.location))
+                }
+            },
             Continue => match self.ctx.loop_data {
                 Some((start, _)) => {
                     self.emit(Instruction::Continue { target: start });
@@ -750,7 +702,7 @@ impl Compiler {
                     Some(v) => {
                         if self.ctx.func == FunctionContext::AsyncFunction
                             && self
-                                .current_code()
+                                .current_codeinfo()
                                 .flags
                                 .contains(bytecode::CodeFlags::IS_GENERATOR)
                         {
@@ -873,7 +825,7 @@ impl Compiler {
         }
 
         let line_number = self.get_source_line_number();
-        self.push_output(CodeObject::new(
+        self.push_output(
             bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED,
             args.posonlyargs_count,
             args.args.len(),
@@ -881,7 +833,7 @@ impl Compiler {
             self.source_path.clone(),
             line_number,
             name.to_owned(),
-        ));
+        );
 
         for name in &args.args {
             self.varname(&name.arg);
@@ -893,7 +845,7 @@ impl Compiler {
         let mut compile_varargs = |va: &ast::Varargs, flag| match va {
             ast::Varargs::None | ast::Varargs::Unnamed => {}
             ast::Varargs::Named(name) => {
-                self.current_code().flags |= flag;
+                self.current_codeinfo().flags |= flag;
                 self.varname(&name.arg);
             }
         };
@@ -925,29 +877,30 @@ impl Compiler {
         orelse: &Option<ast::Suite>,
         finalbody: &Option<ast::Suite>,
     ) -> CompileResult<()> {
-        let mut handler_label = self.new_label();
-        let finally_handler_label = self.new_label();
-        let else_label = self.new_label();
+        let handler_block = self.new_block();
+        let finally_block = self.new_block();
 
         // Setup a finally block if we have a finally statement.
         if finalbody.is_some() {
             self.emit(Instruction::SetupFinally {
-                handler: finally_handler_label,
+                handler: finally_block,
             });
         }
 
+        let else_block = self.new_block();
+
         // try:
         self.emit(Instruction::SetupExcept {
-            handler: handler_label,
+            handler: handler_block,
         });
         self.compile_statements(body)?;
         self.emit(Instruction::PopBlock);
-        self.emit(Instruction::Jump { target: else_label });
+        self.emit(Instruction::Jump { target: else_block });
 
         // except handlers:
-        self.set_label(handler_label);
+        self.switch_to_block(handler_block);
         // Exception is on top of stack now
-        handler_label = self.new_label();
+        let mut next_handler = self.new_block();
         for handler in handlers {
             // If we gave a typ,
             // check if this handler can handle the exception:
@@ -963,7 +916,7 @@ impl Compiler {
 
                 // We cannot handle this exception type:
                 self.emit(Instruction::JumpIfFalse {
-                    target: handler_label,
+                    target: next_handler,
                 });
 
                 // We have a match, store in name (except x as y)
@@ -990,18 +943,18 @@ impl Compiler {
             }
 
             self.emit(Instruction::Jump {
-                target: finally_handler_label,
+                target: finally_block,
             });
 
             // Emit a new label for the next handler
-            self.set_label(handler_label);
-            handler_label = self.new_label();
+            self.switch_to_block(next_handler);
+            next_handler = self.new_block();
         }
 
         self.emit(Instruction::Jump {
-            target: handler_label,
+            target: next_handler,
         });
-        self.set_label(handler_label);
+        self.switch_to_block(next_handler);
         // If code flows here, we have an unhandled exception,
         // raise the exception again!
         self.emit(Instruction::Raise {
@@ -1010,7 +963,7 @@ impl Compiler {
 
         // We successfully ran the try block:
         // else:
-        self.set_label(else_label);
+        self.switch_to_block(else_block);
         if let Some(statements) = orelse {
             self.compile_statements(statements)?;
         }
@@ -1023,7 +976,7 @@ impl Compiler {
         }
 
         // finally:
-        self.set_label(finally_handler_label);
+        self.switch_to_block(finally_block);
         if let Some(statements) = finalbody {
             self.compile_statements(statements)?;
             self.emit(Instruction::EndFinally);
@@ -1269,7 +1222,7 @@ impl Compiler {
 
         self.emit(Instruction::LoadBuildClass);
         let line_number = self.get_source_line_number();
-        self.push_output(CodeObject::new(
+        self.push_output(
             bytecode::CodeFlags::empty(),
             0,
             0,
@@ -1277,7 +1230,7 @@ impl Compiler {
             self.source_path.clone(),
             line_number,
             name.to_owned(),
-        ));
+        );
 
         let (new_body, doc_str) = get_doc(body);
 
@@ -1396,28 +1349,28 @@ impl Compiler {
         body: &[ast::Statement],
         orelse: &Option<Vec<ast::Statement>>,
     ) -> CompileResult<()> {
-        let start_label = self.new_label();
-        let else_label = self.new_label();
-        let end_label = self.new_label();
+        let while_block = self.new_block();
+        let else_block = self.new_block();
+        let after_block = self.new_block();
 
-        self.emit(Instruction::SetupLoop { end: end_label });
-        self.set_label(start_label);
+        self.emit(Instruction::SetupLoop);
+        self.switch_to_block(while_block);
 
-        self.compile_jump_if(test, false, else_label)?;
+        self.compile_jump_if(test, false, else_block)?;
 
         let was_in_loop = self.ctx.loop_data;
-        self.ctx.loop_data = Some((start_label, end_label));
+        self.ctx.loop_data = Some((while_block, after_block));
         self.compile_statements(body)?;
         self.ctx.loop_data = was_in_loop;
         self.emit(Instruction::Jump {
-            target: start_label,
+            target: while_block,
         });
-        self.set_label(else_label);
+        self.switch_to_block(else_block);
         self.emit(Instruction::PopBlock);
         if let Some(orelse) = orelse {
             self.compile_statements(orelse)?;
         }
-        self.set_label(end_label);
+        self.switch_to_block(after_block);
         Ok(())
     }
 
@@ -1430,77 +1383,73 @@ impl Compiler {
         is_async: bool,
     ) -> CompileResult<()> {
         // Start loop
-        let start_label = self.new_label();
-        let else_label = self.new_label();
-        let end_label = self.new_label();
+        let for_block = self.new_block();
+        let else_block = self.new_block();
+        let after_block = self.new_block();
 
-        self.emit(Instruction::SetupLoop { end: end_label });
+        self.emit(Instruction::SetupLoop);
 
         // The thing iterated:
         self.compile_expression(iter)?;
 
         if is_async {
-            let check_asynciter_label = self.new_label();
-            let body_label = self.new_label();
+            let check_asynciter_block = self.new_block();
+            let body_block = self.new_block();
 
             self.emit(Instruction::GetAIter);
 
-            self.set_label(start_label);
+            self.switch_to_block(for_block);
             self.emit(Instruction::SetupExcept {
-                handler: check_asynciter_label,
+                handler: check_asynciter_block,
             });
             self.emit(Instruction::GetANext);
             self.emit_constant(ConstantData::None);
             self.emit(Instruction::YieldFrom);
             self.compile_store(target)?;
             self.emit(Instruction::PopBlock);
-            self.emit(Instruction::Jump { target: body_label });
+            self.emit(Instruction::Jump { target: body_block });
 
-            self.set_label(check_asynciter_label);
+            self.switch_to_block(check_asynciter_block);
             self.emit(Instruction::Duplicate);
             let stopasynciter = self.name("StopAsyncIteration");
             self.emit(Instruction::LoadGlobal(stopasynciter));
             self.emit(Instruction::CompareOperation {
                 op: bytecode::ComparisonOperator::ExceptionMatch,
             });
-            self.emit(Instruction::JumpIfTrue { target: else_label });
+            self.emit(Instruction::JumpIfTrue { target: else_block });
             self.emit(Instruction::Raise {
                 kind: bytecode::RaiseKind::Reraise,
             });
 
-            let was_in_loop = self.ctx.loop_data;
-            self.ctx.loop_data = Some((start_label, end_label));
-            self.set_label(body_label);
-            self.compile_statements(body)?;
-            self.ctx.loop_data = was_in_loop;
+            self.switch_to_block(body_block);
         } else {
             // Retrieve Iterator
             self.emit(Instruction::GetIter);
 
-            self.set_label(start_label);
-            self.emit(Instruction::ForIter { target: else_label });
+            self.switch_to_block(for_block);
+            self.emit(Instruction::ForIter { target: else_block });
 
             // Start of loop iteration, set targets:
             self.compile_store(target)?;
-
-            let was_in_loop = self.ctx.loop_data;
-            self.ctx.loop_data = Some((start_label, end_label));
-            self.compile_statements(body)?;
-            self.ctx.loop_data = was_in_loop;
         }
 
-        self.emit(Instruction::Jump {
-            target: start_label,
-        });
-        self.set_label(else_label);
+        let was_in_loop = self.ctx.loop_data;
+        self.ctx.loop_data = Some((for_block, after_block));
+        self.compile_statements(body)?;
+        self.ctx.loop_data = was_in_loop;
+        self.emit(Instruction::Jump { target: for_block });
+
+        self.switch_to_block(else_block);
         self.emit(Instruction::PopBlock);
         if let Some(orelse) = orelse {
             self.compile_statements(orelse)?;
         }
-        self.set_label(end_label);
+
+        self.switch_to_block(after_block);
         if is_async {
             self.emit(Instruction::Pop);
         }
+
         Ok(())
     }
 
@@ -1536,8 +1485,13 @@ impl Compiler {
         // initialize lhs outside of loop
         self.compile_expression(&vals[0])?;
 
-        let break_label = self.new_label();
-        let last_label = self.new_label();
+        let end_blocks = if vals.len() > 2 {
+            let break_block = self.new_block();
+            let after_block = self.new_block();
+            Some((break_block, after_block))
+        } else {
+            None
+        };
 
         // for all comparisons except the last (as the last one doesn't need a conditional jump)
         let ops_slice = &ops[0..ops.len()];
@@ -1553,9 +1507,11 @@ impl Compiler {
             });
 
             // if comparison result is false, we break with this value; if true, try the next one.
-            self.emit(Instruction::JumpIfFalseOrPop {
-                target: break_label,
-            });
+            if let Some((break_block, _)) = end_blocks {
+                self.emit(Instruction::JumpIfFalseOrPop {
+                    target: break_block,
+                });
+            }
         }
 
         // handle the last comparison
@@ -1564,15 +1520,17 @@ impl Compiler {
             op: to_operator(ops.last().unwrap()),
         });
 
-        if vals.len() > 2 {
-            self.emit(Instruction::Jump { target: last_label });
+        if let Some((break_block, after_block)) = end_blocks {
+            self.emit(Instruction::Jump {
+                target: after_block,
+            });
 
             // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs.
-            self.set_label(break_label);
+            self.switch_to_block(break_block);
             self.emit(Instruction::Rotate { amount: 2 });
             self.emit(Instruction::Pop);
 
-            self.set_label(last_label);
+            self.switch_to_block(after_block);
         }
 
         Ok(())
@@ -1715,7 +1673,7 @@ impl Compiler {
         &mut self,
         expression: &ast::Expression,
         condition: bool,
-        target_label: Label,
+        target_block: ir::BlockIdx,
     ) -> CompileResult<()> {
         // Compile expression for test, and jump to label if false
         match &expression.node {
@@ -1724,21 +1682,21 @@ impl Compiler {
                     ast::BooleanOperator::And => {
                         if condition {
                             // If all values are true.
-                            let end_label = self.new_label();
+                            let end_block = self.new_block();
                             let (last_value, values) = values.split_last().unwrap();
 
                             // If any of the values is false, we can short-circuit.
                             for value in values {
-                                self.compile_jump_if(value, false, end_label)?;
+                                self.compile_jump_if(value, false, end_block)?;
                             }
 
                             // It depends upon the last value now: will it be true?
-                            self.compile_jump_if(last_value, true, target_label)?;
-                            self.set_label(end_label);
+                            self.compile_jump_if(last_value, true, target_block)?;
+                            self.switch_to_block(end_block);
                         } else {
                             // If any value is false, the whole condition is false.
                             for value in values {
-                                self.compile_jump_if(value, false, target_label)?;
+                                self.compile_jump_if(value, false, target_block)?;
                             }
                         }
                     }
@@ -1746,21 +1704,21 @@ impl Compiler {
                         if condition {
                             // If any of the values is true.
                             for value in values {
-                                self.compile_jump_if(value, true, target_label)?;
+                                self.compile_jump_if(value, true, target_block)?;
                             }
                         } else {
                             // If all of the values are false.
-                            let end_label = self.new_label();
+                            let end_block = self.new_block();
                             let (last_value, values) = values.split_last().unwrap();
 
                             // If any value is true, we can short-circuit:
                             for value in values {
-                                self.compile_jump_if(value, true, end_label)?;
+                                self.compile_jump_if(value, true, end_block)?;
                             }
 
                             // It all depends upon the last value now!
-                            self.compile_jump_if(last_value, false, target_label)?;
-                            self.set_label(end_label);
+                            self.compile_jump_if(last_value, false, target_block)?;
+                            self.switch_to_block(end_block);
                         }
                     }
                 }
@@ -1769,18 +1727,18 @@ impl Compiler {
                 op: ast::UnaryOperator::Not,
                 a,
             } => {
-                self.compile_jump_if(a, !condition, target_label)?;
+                self.compile_jump_if(a, !condition, target_block)?;
             }
             _ => {
                 // Fall back case which always will work!
                 self.compile_expression(expression)?;
                 if condition {
                     self.emit(Instruction::JumpIfTrue {
-                        target: target_label,
+                        target: target_block,
                     });
                 } else {
                     self.emit(Instruction::JumpIfFalse {
-                        target: target_label,
+                        target: target_block,
                     });
                 }
             }
@@ -1795,7 +1753,7 @@ impl Compiler {
         op: &ast::BooleanOperator,
         values: &[ast::Expression],
     ) -> CompileResult<()> {
-        let end_label = self.new_label();
+        let after_block = self.new_block();
 
         let (last_value, values) = values.split_last().unwrap();
         for value in values {
@@ -1803,17 +1761,21 @@ impl Compiler {
 
             match op {
                 ast::BooleanOperator::And => {
-                    self.emit(Instruction::JumpIfFalseOrPop { target: end_label });
+                    self.emit(Instruction::JumpIfFalseOrPop {
+                        target: after_block,
+                    });
                 }
                 ast::BooleanOperator::Or => {
-                    self.emit(Instruction::JumpIfTrueOrPop { target: end_label });
+                    self.emit(Instruction::JumpIfTrueOrPop {
+                        target: after_block,
+                    });
                 }
             }
         }
 
         // If all values did not qualify, take the value of the last value:
         self.compile_expression(last_value)?;
-        self.set_label(end_label);
+        self.switch_to_block(after_block);
         Ok(())
     }
 
@@ -2046,17 +2008,22 @@ impl Compiler {
                 return Err(self.error(CompileErrorType::InvalidStarExpr));
             }
             IfExpression { test, body, orelse } => {
-                let no_label = self.new_label();
-                let end_label = self.new_label();
-                self.compile_jump_if(test, false, no_label)?;
+                let else_block = self.new_block();
+                let after_block = self.new_block();
+                self.compile_jump_if(test, false, else_block)?;
+
                 // True case
                 self.compile_expression(body)?;
-                self.emit(Instruction::Jump { target: end_label });
+                self.emit(Instruction::Jump {
+                    target: after_block,
+                });
+
                 // False case
-                self.set_label(no_label);
+                self.switch_to_block(else_block);
                 self.compile_expression(orelse)?;
+
                 // End
-                self.set_label(end_label);
+                self.switch_to_block(after_block);
             }
 
             NamedExpression { left, right } => {
@@ -2210,7 +2177,7 @@ impl Compiler {
 
         let line_number = self.get_source_line_number();
         // Create magnificent function <listcomp>:
-        self.push_output(CodeObject::new(
+        self.push_output(
             bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED,
             1,
             1,
@@ -2218,7 +2185,7 @@ impl Compiler {
             self.source_path.clone(),
             line_number,
             name.clone(),
-        ));
+        );
         let arg0 = self.varname(".0");
 
         // Create empty object of proper type:
@@ -2251,6 +2218,9 @@ impl Compiler {
                 unimplemented!("async for comprehensions");
             }
 
+            // Setup for loop:
+            self.emit(Instruction::SetupLoop);
+
             if loop_labels.is_empty() {
                 // Load iterator onto stack (passed as first argument):
                 self.emit(Instruction::LoadFast(arg0));
@@ -2262,19 +2232,20 @@ impl Compiler {
                 self.emit(Instruction::GetIter);
             }
 
-            // Setup for loop:
-            let start_label = self.new_label();
-            let end_label = self.new_label();
-            loop_labels.push((start_label, end_label));
-            self.emit(Instruction::SetupLoop { end: end_label });
-            self.set_label(start_label);
-            self.emit(Instruction::ForIter { target: end_label });
+            let loop_block = self.new_block();
+            let after_block = self.new_block();
+            loop_labels.push((loop_block, after_block));
+
+            self.switch_to_block(loop_block);
+            self.emit(Instruction::ForIter {
+                target: after_block,
+            });
 
             self.compile_store(&generator.target)?;
 
             // Now evaluate the ifs:
             for if_condition in &generator.ifs {
-                self.compile_jump_if(if_condition, false, start_label)?
+                self.compile_jump_if(if_condition, false, loop_block)?
             }
         }
 
@@ -2320,14 +2291,12 @@ impl Compiler {
             }
         }
 
-        for (start_label, end_label) in loop_labels.iter().rev() {
+        for (loop_block, after_block) in loop_labels.iter().rev().copied() {
             // Repeat:
-            self.emit(Instruction::Jump {
-                target: *start_label,
-            });
+            self.emit(Instruction::Jump { target: loop_block });
 
             // End of for loop:
-            self.set_label(*end_label);
+            self.switch_to_block(after_block);
             self.emit(Instruction::PopBlock);
         }
 
@@ -2428,14 +2397,16 @@ impl Compiler {
     }
 
     // Low level helper functions:
-    fn emit(&mut self, instruction: Instruction) {
+    fn emit(&mut self, instr: Instruction) {
         let location = compile_location(&self.current_source_location);
         // TODO: insert source filename
-        let info = self.current_codeinfo();
-        info.instructions.push(instruction);
-        info.locations.push(location);
+        self.current_block()
+            .instructions
+            .push(ir::InstructionInfo { instr, location });
     }
 
+    // fn block_done()
+
     fn emit_constant(&mut self, constant: ConstantData) {
         let info = self.current_codeinfo();
         let idx = info.constants.len() as u32;
@@ -2443,35 +2414,31 @@ impl Compiler {
         self.emit(Instruction::LoadConst { idx })
     }
 
-    fn current_code(&mut self) -> &mut CodeObject {
-        &mut self.current_codeinfo().code
-    }
-
     fn current_codeinfo(&mut self) -> &mut CodeInfo {
         self.code_stack.last_mut().expect("no code on stack")
     }
 
-    // Generate a new label
-    fn new_label(&mut self) -> Label {
-        let label_map = &mut self.current_codeinfo().label_map;
-        let label = Label(label_map.len() as u32);
-        label_map.push(Label(u32::MAX));
-        label
+    fn current_block(&mut self) -> &mut ir::Block {
+        let info = self.current_codeinfo();
+        &mut info.blocks[info.block_order.last().unwrap().0 as usize]
+    }
+
+    fn new_block(&mut self) -> ir::BlockIdx {
+        let code = self.current_codeinfo();
+        let idx = bytecode::Label(code.blocks.len() as u32);
+        code.blocks.push(ir::Block::default());
+        idx
     }
 
-    // Assign current position the given label
-    fn set_label(&mut self, label: Label) {
-        let CodeInfo {
-            instructions,
-            label_map,
-            ..
-        } = self.current_codeinfo();
-        let actual_label = Label(instructions.len() as u32);
-        let prev_val = std::mem::replace(&mut label_map[label.0 as usize], actual_label);
+    fn switch_to_block(&mut self, block: ir::BlockIdx) {
+        let code = self.current_codeinfo();
+        let last = code.block_order.last().unwrap();
+        code.blocks[last.0 as usize].done = true;
         debug_assert!(
-            prev_val.0 == u32::MAX || prev_val == actual_label,
-            "double-set a label"
+            !code.blocks[block.0 as usize].done,
+            "switching to done block"
         );
+        code.block_order.push(block);
     }
 
     fn set_source_location(&mut self, location: ast::Location) {
@@ -2491,7 +2458,7 @@ impl Compiler {
     }
 
     fn mark_generator(&mut self) {
-        self.current_code().flags |= bytecode::CodeFlags::IS_GENERATOR
+        self.current_codeinfo().flags |= bytecode::CodeFlags::IS_GENERATOR
     }
 }
 
diff --git a/src/ir.rs b/src/ir.rs
new file mode 100644
index 00000000..ce4877a5
--- /dev/null
+++ b/src/ir.rs
@@ -0,0 +1,224 @@
+use indexmap::IndexSet;
+use rustpython_bytecode::{CodeFlags, CodeObject, ConstantData, Instruction, Label, Location};
+
+pub type BlockIdx = Label;
+
+pub struct InstructionInfo {
+    /// If the instruction has a Label argument, it's actually a BlockIdx, not a code offset
+    pub instr: Instruction,
+    pub location: Location,
+}
+
+pub struct Block {
+    pub instructions: Vec<InstructionInfo>,
+    pub done: bool,
+}
+impl Default for Block {
+    fn default() -> Self {
+        Block {
+            instructions: Vec::new(),
+            done: false,
+        }
+    }
+}
+
+pub struct CodeInfo {
+    pub flags: CodeFlags,
+    pub posonlyarg_count: usize, // Number of positional-only arguments
+    pub arg_count: usize,
+    pub kwonlyarg_count: usize,
+    pub source_path: String,
+    pub first_line_number: usize,
+    pub obj_name: String, // Name of the object that created this code object
+
+    pub blocks: Vec<Block>,
+    pub block_order: Vec<BlockIdx>,
+    pub constants: Vec<ConstantData>,
+    pub name_cache: IndexSet<String>,
+    pub varname_cache: IndexSet<String>,
+    pub cellvar_cache: IndexSet<String>,
+    pub freevar_cache: IndexSet<String>,
+}
+impl CodeInfo {
+    pub fn finalize_code(mut self, optimize: u8) -> CodeObject {
+        let max_stacksize = self.max_stacksize();
+        let cell2arg = self.cell2arg();
+
+        if optimize > 0 {
+            self.dce();
+        }
+
+        let CodeInfo {
+            flags,
+            posonlyarg_count,
+            arg_count,
+            kwonlyarg_count,
+            source_path,
+            first_line_number,
+            obj_name,
+
+            mut blocks,
+            block_order,
+            constants,
+            name_cache,
+            varname_cache,
+            cellvar_cache,
+            freevar_cache,
+        } = self;
+
+        assert!(block_order.len() == blocks.len());
+
+        let mut num_instructions = 0;
+        let mut block_to_offset = vec![Label(0); blocks.len()];
+
+        for idx in &block_order {
+            let idx = idx.0 as usize;
+            block_to_offset[idx] = Label(num_instructions as u32);
+            num_instructions += blocks[idx].instructions.len();
+        }
+
+        let mut instructions = Vec::with_capacity(num_instructions);
+        let mut locations = Vec::with_capacity(num_instructions);
+
+        for idx in block_order {
+            let block = std::mem::take(&mut blocks[idx.0 as usize]);
+            for mut instr in block.instructions {
+                if let Some(l) = instr.instr.label_arg_mut() {
+                    *l = block_to_offset[l.0 as usize];
+                }
+                instructions.push(instr.instr);
+                locations.push(instr.location);
+            }
+        }
+
+        CodeObject {
+            flags,
+            posonlyarg_count,
+            arg_count,
+            kwonlyarg_count,
+            source_path,
+            first_line_number,
+            obj_name,
+
+            max_stacksize,
+            instructions: instructions.into_boxed_slice(),
+            locations: locations.into_boxed_slice(),
+            constants: constants.into(),
+            names: name_cache.into_iter().collect(),
+            varnames: varname_cache.into_iter().collect(),
+            cellvars: cellvar_cache.into_iter().collect(),
+            freevars: freevar_cache.into_iter().collect(),
+            cell2arg,
+        }
+    }
+
+    fn cell2arg(&self) -> Option<Box<[isize]>> {
+        if self.cellvar_cache.is_empty() {
+            return None;
+        }
+
+        let total_args = self.arg_count
+            + self.kwonlyarg_count
+            + self.flags.contains(CodeFlags::HAS_VARARGS) as usize
+            + self.flags.contains(CodeFlags::HAS_VARKEYWORDS) as usize;
+
+        let mut found_cellarg = false;
+        let cell2arg = self
+            .cellvar_cache
+            .iter()
+            .map(|var| {
+                self.varname_cache
+                    .get_index_of(var)
+                    // check that it's actually an arg
+                    .filter(|i| *i < total_args)
+                    .map_or(-1, |i| {
+                        found_cellarg = true;
+                        i as isize
+                    })
+            })
+            .collect::<Box<[_]>>();
+
+        if found_cellarg {
+            Some(cell2arg)
+        } else {
+            None
+        }
+    }
+
+    fn dce(&mut self) {
+        for block in &mut self.blocks {
+            let mut last_instr = None;
+            for (i, ins) in block.instructions.iter().enumerate() {
+                if ins.instr.unconditional_branch() {
+                    last_instr = Some(i);
+                    break;
+                }
+            }
+            if let Some(i) = last_instr {
+                block.instructions.truncate(i + 1);
+            }
+        }
+    }
+
+    fn max_stacksize(&self) -> u32 {
+        let mut maxdepth = 0;
+        let mut stack = Vec::with_capacity(self.blocks.len());
+        let mut startdepths = vec![0; self.blocks.len()];
+        stack.push((Label(0), 0));
+        'process_blocks: while let Some((block, blockorder)) = stack.pop() {
+            let mut depth = startdepths[block.0 as usize];
+            for i in &self.blocks[block.0 as usize].instructions {
+                let instr = &i.instr;
+                let effect = instr.stack_effect(false);
+                let new_depth = depth + effect;
+                if new_depth > maxdepth {
+                    maxdepth = new_depth
+                }
+                // we don't want to worry about Continue or Break, they use unwinding to jump to
+                // their targets and as such the stack size is taken care of in frame.rs by setting
+                // it back to the level it was at when SetupLoop was run
+                let jump_label = instr.label_arg().filter(
+                    |_| !matches!(instr, Instruction::Continue { .. } | Instruction::Break { .. }),
+                );
+                if let Some(&target_block) = jump_label {
+                    let effect = instr.stack_effect(true);
+                    let target_depth = depth + effect;
+                    if target_depth > maxdepth {
+                        maxdepth = target_depth
+                    }
+                    stackdepth_push(
+                        &mut stack,
+                        &mut startdepths,
+                        (target_block, u32::MAX),
+                        target_depth,
+                    );
+                }
+                depth = new_depth;
+                if instr.unconditional_branch() {
+                    continue 'process_blocks;
+                }
+            }
+            let next_blockorder = if blockorder == u32::MAX {
+                self.block_order.iter().position(|x| *x == block).unwrap() as u32 + 1
+            } else {
+                blockorder + 1
+            };
+            let next = self.block_order[next_blockorder as usize];
+            stackdepth_push(&mut stack, &mut startdepths, (next, next_blockorder), depth);
+        }
+        maxdepth as u32
+    }
+}
+
+fn stackdepth_push(
+    stack: &mut Vec<(Label, u32)>,
+    startdepths: &mut [i32],
+    target: (Label, u32),
+    depth: i32,
+) {
+    let block_depth = &mut startdepths[target.0 .0 as usize];
+    if depth > *block_depth {
+        *block_depth = depth;
+        stack.push(target);
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index ab299864..523f92e8 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -7,5 +7,6 @@ extern crate log;
 
 pub mod compile;
 pub mod error;
+pub mod ir;
 pub mod mode;
 pub mod symboltable;

From d567f967e84d1666ec50f1fcfd3ff8da479ce6e4 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Fri, 18 Dec 2020 13:30:42 -0600
Subject: [PATCH 181/459] Use boxvec for frame.stack

---
 src/compile.rs | 54 ++++++++++++++++++++------------------------------
 src/ir.rs      | 28 ++++++++++++++++++--------
 2 files changed, 42 insertions(+), 40 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 739be8ed..18b3eb9f 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -178,10 +178,11 @@ impl Compiler {
         posonlyarg_count: usize,
         arg_count: usize,
         kwonlyarg_count: usize,
-        source_path: String,
-        first_line_number: usize,
         obj_name: String,
     ) {
+        let source_path = self.source_path.clone();
+        let first_line_number = self.get_source_line_number();
+
         let table = self
             .symbol_table_stack
             .last_mut()
@@ -824,14 +825,11 @@ impl Compiler {
             funcflags |= bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS;
         }
 
-        let line_number = self.get_source_line_number();
         self.push_output(
             bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED,
             args.posonlyargs_count,
             args.args.len(),
             args.kwonlyargs.len(),
-            self.source_path.clone(),
-            line_number,
             name.to_owned(),
         );
 
@@ -900,8 +898,9 @@ impl Compiler {
         // except handlers:
         self.switch_to_block(handler_block);
         // Exception is on top of stack now
-        let mut next_handler = self.new_block();
         for handler in handlers {
+            let next_handler = self.new_block();
+
             // If we gave a typ,
             // check if this handler can handle the exception:
             if let Some(exc_type) = &handler.typ {
@@ -937,7 +936,7 @@ impl Compiler {
             self.emit(Instruction::PopException);
 
             if finalbody.is_some() {
-                self.emit(Instruction::PopBlock); // pop finally block
+                self.emit(Instruction::PopBlock); // pop excepthandler block
                                                   // We enter the finally block, without exception.
                 self.emit(Instruction::EnterFinally);
             }
@@ -948,13 +947,8 @@ impl Compiler {
 
             // Emit a new label for the next handler
             self.switch_to_block(next_handler);
-            next_handler = self.new_block();
         }
 
-        self.emit(Instruction::Jump {
-            target: next_handler,
-        });
-        self.switch_to_block(next_handler);
         // If code flows here, we have an unhandled exception,
         // raise the exception again!
         self.emit(Instruction::Raise {
@@ -971,7 +965,7 @@ impl Compiler {
         if finalbody.is_some() {
             self.emit(Instruction::PopBlock); // pop finally block
 
-            // We enter the finally block, without return / exception.
+            // We enter the finallyhandler block, without return / exception.
             self.emit(Instruction::EnterFinally);
         }
 
@@ -998,6 +992,9 @@ impl Compiler {
 
         self.prepare_decorators(decorator_list)?;
         let mut funcflags = self.enter_function(name, args)?;
+        self.current_codeinfo()
+            .flags
+            .set(bytecode::CodeFlags::IS_COROUTINE, is_async);
 
         // remember to restore self.ctx.in_loop to the original after the function is compiled
         let prev_ctx = self.ctx;
@@ -1031,7 +1028,7 @@ impl Compiler {
             }
         }
 
-        let mut code = self.pop_code_object();
+        let code = self.pop_code_object();
         self.current_qualified_path = old_qualified_path;
         self.ctx = prev_ctx;
 
@@ -1081,10 +1078,6 @@ impl Compiler {
             });
         }
 
-        if is_async {
-            code.flags |= bytecode::CodeFlags::IS_COROUTINE;
-        }
-
         if self.build_closure(&code) {
             funcflags |= bytecode::MakeFunctionFlags::CLOSURE;
         }
@@ -1221,16 +1214,7 @@ impl Compiler {
         self.current_qualified_path = Some(qualified_name.clone());
 
         self.emit(Instruction::LoadBuildClass);
-        let line_number = self.get_source_line_number();
-        self.push_output(
-            bytecode::CodeFlags::empty(),
-            0,
-            0,
-            0,
-            self.source_path.clone(),
-            line_number,
-            name.to_owned(),
-        );
+        self.push_output(bytecode::CodeFlags::empty(), 0, 0, 0, name.to_owned());
 
         let (new_body, doc_str) = get_doc(body);
 
@@ -2175,22 +2159,24 @@ impl Compiler {
         }
         .to_owned();
 
-        let line_number = self.get_source_line_number();
         // Create magnificent function <listcomp>:
         self.push_output(
             bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED,
             1,
             1,
             0,
-            self.source_path.clone(),
-            line_number,
             name.clone(),
         );
         let arg0 = self.varname(".0");
 
+        let is_genexpr = matches!(kind, ast::ComprehensionKind::GeneratorExpression { .. });
+
         // Create empty object of proper type:
         match kind {
-            ast::ComprehensionKind::GeneratorExpression { .. } => {}
+            ast::ComprehensionKind::GeneratorExpression { .. } => {
+                // pop the None that was passed to kickstart the generator
+                self.emit(Instruction::Pop);
+            }
             ast::ComprehensionKind::List { .. } => {
                 self.emit(Instruction::BuildList {
                     size: 0,
@@ -2300,6 +2286,10 @@ impl Compiler {
             self.emit(Instruction::PopBlock);
         }
 
+        if is_genexpr {
+            self.emit_constant(ConstantData::None)
+        }
+
         // Return freshly filled list:
         self.emit(Instruction::ReturnValue);
 
diff --git a/src/ir.rs b/src/ir.rs
index ce4877a5..6b564e07 100644
--- a/src/ir.rs
+++ b/src/ir.rs
@@ -161,16 +161,20 @@ impl CodeInfo {
     }
 
     fn max_stacksize(&self) -> u32 {
-        let mut maxdepth = 0;
+        let mut maxdepth = 0u32;
         let mut stack = Vec::with_capacity(self.blocks.len());
-        let mut startdepths = vec![0; self.blocks.len()];
+        let mut startdepths = vec![u32::MAX; self.blocks.len()];
+        // if it's a generator or a coroutine, it starts with something already on the stack
+        startdepths[0] =
+            self.flags
+                .intersects(CodeFlags::IS_GENERATOR | CodeFlags::IS_COROUTINE) as u32;
         stack.push((Label(0), 0));
         'process_blocks: while let Some((block, blockorder)) = stack.pop() {
             let mut depth = startdepths[block.0 as usize];
             for i in &self.blocks[block.0 as usize].instructions {
                 let instr = &i.instr;
                 let effect = instr.stack_effect(false);
-                let new_depth = depth + effect;
+                let new_depth = add_ui(depth, effect);
                 if new_depth > maxdepth {
                     maxdepth = new_depth
                 }
@@ -182,7 +186,7 @@ impl CodeInfo {
                 );
                 if let Some(&target_block) = jump_label {
                     let effect = instr.stack_effect(true);
-                    let target_depth = depth + effect;
+                    let target_depth = add_ui(depth, effect);
                     if target_depth > maxdepth {
                         maxdepth = target_depth
                     }
@@ -206,19 +210,27 @@ impl CodeInfo {
             let next = self.block_order[next_blockorder as usize];
             stackdepth_push(&mut stack, &mut startdepths, (next, next_blockorder), depth);
         }
-        maxdepth as u32
+        maxdepth
     }
 }
 
 fn stackdepth_push(
     stack: &mut Vec<(Label, u32)>,
-    startdepths: &mut [i32],
+    startdepths: &mut [u32],
     target: (Label, u32),
-    depth: i32,
+    depth: u32,
 ) {
     let block_depth = &mut startdepths[target.0 .0 as usize];
-    if depth > *block_depth {
+    if *block_depth == u32::MAX || depth > *block_depth {
         *block_depth = depth;
         stack.push(target);
     }
 }
+
+fn add_ui(a: u32, b: i32) -> u32 {
+    if b < 0 {
+        a - b.abs() as u32
+    } else {
+        a + b as u32
+    }
+}

From a2c0db23f65bff36ee19045ae3a79ee54382f92d Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 21 Dec 2020 18:39:28 -0600
Subject: [PATCH 182/459] Don't keep a separate blockorder vec

---
 src/compile.rs | 25 +++++++++++-------
 src/ir.rs      | 71 ++++++++++++++++++++++----------------------------
 2 files changed, 47 insertions(+), 49 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 18b3eb9f..3de5fc2f 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -138,7 +138,7 @@ impl Compiler {
             obj_name: code_name,
 
             blocks: vec![ir::Block::default()],
-            block_order: vec![bytecode::Label(0)],
+            current_block: bytecode::Label(0),
             constants: Vec::new(),
             name_cache: IndexSet::new(),
             varname_cache: IndexSet::new(),
@@ -215,7 +215,7 @@ impl Compiler {
             obj_name,
 
             blocks: vec![ir::Block::default()],
-            block_order: vec![bytecode::Label(0)],
+            current_block: bytecode::Label(0),
             constants: Vec::new(),
             name_cache: IndexSet::new(),
             varname_cache: IndexSet::new(),
@@ -2410,7 +2410,7 @@ impl Compiler {
 
     fn current_block(&mut self) -> &mut ir::Block {
         let info = self.current_codeinfo();
-        &mut info.blocks[info.block_order.last().unwrap().0 as usize]
+        &mut info.blocks[info.current_block.0 as usize]
     }
 
     fn new_block(&mut self) -> ir::BlockIdx {
@@ -2422,13 +2422,20 @@ impl Compiler {
 
     fn switch_to_block(&mut self, block: ir::BlockIdx) {
         let code = self.current_codeinfo();
-        let last = code.block_order.last().unwrap();
-        code.blocks[last.0 as usize].done = true;
-        debug_assert!(
-            !code.blocks[block.0 as usize].done,
-            "switching to done block"
+        let prev = code.current_block;
+        assert_eq!(
+            code.blocks[block.0 as usize].next.0,
+            u32::MAX,
+            "switching to completed block"
         );
-        code.block_order.push(block);
+        let prev_block = &mut code.blocks[prev.0 as usize];
+        assert_eq!(
+            prev_block.next.0,
+            u32::MAX,
+            "switching from block that's already got a next"
+        );
+        prev_block.next = block;
+        code.current_block = block;
     }
 
     fn set_source_location(&mut self, location: ast::Location) {
diff --git a/src/ir.rs b/src/ir.rs
index 6b564e07..dde8962a 100644
--- a/src/ir.rs
+++ b/src/ir.rs
@@ -3,21 +3,25 @@ use rustpython_bytecode::{CodeFlags, CodeObject, ConstantData, Instruction, Labe
 
 pub type BlockIdx = Label;
 
+#[derive(Debug)]
 pub struct InstructionInfo {
     /// If the instruction has a Label argument, it's actually a BlockIdx, not a code offset
     pub instr: Instruction,
     pub location: Location,
 }
 
+// TODO: look into using petgraph for handling blocks and stuff? it's heavier than this, but it
+// might enable more analysis/optimizations
+#[derive(Debug)]
 pub struct Block {
     pub instructions: Vec<InstructionInfo>,
-    pub done: bool,
+    pub next: BlockIdx,
 }
 impl Default for Block {
     fn default() -> Self {
         Block {
             instructions: Vec::new(),
-            done: false,
+            next: Label(u32::MAX),
         }
     }
 }
@@ -32,7 +36,7 @@ pub struct CodeInfo {
     pub obj_name: String, // Name of the object that created this code object
 
     pub blocks: Vec<Block>,
-    pub block_order: Vec<BlockIdx>,
+    pub current_block: BlockIdx,
     pub constants: Vec<ConstantData>,
     pub name_cache: IndexSet<String>,
     pub varname_cache: IndexSet<String>,
@@ -57,8 +61,8 @@ impl CodeInfo {
             first_line_number,
             obj_name,
 
-            mut blocks,
-            block_order,
+            blocks,
+            current_block: _,
             constants,
             name_cache,
             varname_cache,
@@ -66,28 +70,25 @@ impl CodeInfo {
             freevar_cache,
         } = self;
 
-        assert!(block_order.len() == blocks.len());
-
         let mut num_instructions = 0;
         let mut block_to_offset = vec![Label(0); blocks.len()];
 
-        for idx in &block_order {
-            let idx = idx.0 as usize;
-            block_to_offset[idx] = Label(num_instructions as u32);
-            num_instructions += blocks[idx].instructions.len();
+        for (idx, block) in iter_blocks(&blocks) {
+            block_to_offset[idx.0 as usize] = Label(num_instructions as u32);
+            num_instructions += block.instructions.len();
         }
 
         let mut instructions = Vec::with_capacity(num_instructions);
         let mut locations = Vec::with_capacity(num_instructions);
 
-        for idx in block_order {
-            let block = std::mem::take(&mut blocks[idx.0 as usize]);
-            for mut instr in block.instructions {
-                if let Some(l) = instr.instr.label_arg_mut() {
+        for (_, block) in iter_blocks(&blocks) {
+            for info in &block.instructions {
+                let mut instr = info.instr.clone();
+                if let Some(l) = instr.label_arg_mut() {
                     *l = block_to_offset[l.0 as usize];
                 }
-                instructions.push(instr.instr);
-                locations.push(instr.location);
+                instructions.push(instr);
+                locations.push(info.location);
             }
         }
 
@@ -168,10 +169,11 @@ impl CodeInfo {
         startdepths[0] =
             self.flags
                 .intersects(CodeFlags::IS_GENERATOR | CodeFlags::IS_COROUTINE) as u32;
-        stack.push((Label(0), 0));
-        'process_blocks: while let Some((block, blockorder)) = stack.pop() {
+        stack.push(Label(0));
+        'process_blocks: while let Some(block) = stack.pop() {
             let mut depth = startdepths[block.0 as usize];
-            for i in &self.blocks[block.0 as usize].instructions {
+            let block = &self.blocks[block.0 as usize];
+            for i in &block.instructions {
                 let instr = &i.instr;
                 let effect = instr.stack_effect(false);
                 let new_depth = add_ui(depth, effect);
@@ -190,37 +192,21 @@ impl CodeInfo {
                     if target_depth > maxdepth {
                         maxdepth = target_depth
                     }
-                    stackdepth_push(
-                        &mut stack,
-                        &mut startdepths,
-                        (target_block, u32::MAX),
-                        target_depth,
-                    );
+                    stackdepth_push(&mut stack, &mut startdepths, target_block, target_depth);
                 }
                 depth = new_depth;
                 if instr.unconditional_branch() {
                     continue 'process_blocks;
                 }
             }
-            let next_blockorder = if blockorder == u32::MAX {
-                self.block_order.iter().position(|x| *x == block).unwrap() as u32 + 1
-            } else {
-                blockorder + 1
-            };
-            let next = self.block_order[next_blockorder as usize];
-            stackdepth_push(&mut stack, &mut startdepths, (next, next_blockorder), depth);
+            stackdepth_push(&mut stack, &mut startdepths, block.next, depth);
         }
         maxdepth
     }
 }
 
-fn stackdepth_push(
-    stack: &mut Vec<(Label, u32)>,
-    startdepths: &mut [u32],
-    target: (Label, u32),
-    depth: u32,
-) {
-    let block_depth = &mut startdepths[target.0 .0 as usize];
+fn stackdepth_push(stack: &mut Vec<Label>, startdepths: &mut [u32], target: Label, depth: u32) {
+    let block_depth = &mut startdepths[target.0 as usize];
     if *block_depth == u32::MAX || depth > *block_depth {
         *block_depth = depth;
         stack.push(target);
@@ -234,3 +220,8 @@ fn add_ui(a: u32, b: i32) -> u32 {
         a + b as u32
     }
 }
+
+fn iter_blocks(blocks: &[Block]) -> impl Iterator<Item = (BlockIdx, &Block)> + '_ {
+    let get_idx = move |i: BlockIdx| blocks.get(i.0 as usize).map(|b| (i, b));
+    std::iter::successors(get_idx(Label(0)), move |(_, b)| get_idx(b.next)) // if b.next is u32::MAX that's the end
+}

From 64fa9ccba96f63083cba56343e6b0f88d590a7b2 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 21 Dec 2020 19:03:49 -0600
Subject: [PATCH 183/459] Don't push something on the stack when starting a
 generator

---
 src/compile.rs | 36 ++++++++++++------------------------
 src/ir.rs      |  5 +----
 2 files changed, 13 insertions(+), 28 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 3de5fc2f..d9a1c3aa 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1376,9 +1376,8 @@ impl Compiler {
         // The thing iterated:
         self.compile_expression(iter)?;
 
-        if is_async {
+        let check_asynciter_block = if is_async {
             let check_asynciter_block = self.new_block();
-            let body_block = self.new_block();
 
             self.emit(Instruction::GetAIter);
 
@@ -1391,21 +1390,8 @@ impl Compiler {
             self.emit(Instruction::YieldFrom);
             self.compile_store(target)?;
             self.emit(Instruction::PopBlock);
-            self.emit(Instruction::Jump { target: body_block });
 
-            self.switch_to_block(check_asynciter_block);
-            self.emit(Instruction::Duplicate);
-            let stopasynciter = self.name("StopAsyncIteration");
-            self.emit(Instruction::LoadGlobal(stopasynciter));
-            self.emit(Instruction::CompareOperation {
-                op: bytecode::ComparisonOperator::ExceptionMatch,
-            });
-            self.emit(Instruction::JumpIfTrue { target: else_block });
-            self.emit(Instruction::Raise {
-                kind: bytecode::RaiseKind::Reraise,
-            });
-
-            self.switch_to_block(body_block);
+            Some(check_asynciter_block)
         } else {
             // Retrieve Iterator
             self.emit(Instruction::GetIter);
@@ -1415,7 +1401,8 @@ impl Compiler {
 
             // Start of loop iteration, set targets:
             self.compile_store(target)?;
-        }
+            None
+        };
 
         let was_in_loop = self.ctx.loop_data;
         self.ctx.loop_data = Some((for_block, after_block));
@@ -1423,6 +1410,11 @@ impl Compiler {
         self.ctx.loop_data = was_in_loop;
         self.emit(Instruction::Jump { target: for_block });
 
+        if let Some(check_asynciter_block) = check_asynciter_block {
+            self.switch_to_block(check_asynciter_block);
+            self.emit(Instruction::EndAsyncFor);
+        }
+
         self.switch_to_block(else_block);
         self.emit(Instruction::PopBlock);
         if let Some(orelse) = orelse {
@@ -1430,9 +1422,6 @@ impl Compiler {
         }
 
         self.switch_to_block(after_block);
-        if is_async {
-            self.emit(Instruction::Pop);
-        }
 
         Ok(())
     }
@@ -2169,14 +2158,13 @@ impl Compiler {
         );
         let arg0 = self.varname(".0");
 
+        // if this is a generator expression, we need to insert a LoadConst(None) before we return;
+        // the other kinds load their collection types below
         let is_genexpr = matches!(kind, ast::ComprehensionKind::GeneratorExpression { .. });
 
         // Create empty object of proper type:
         match kind {
-            ast::ComprehensionKind::GeneratorExpression { .. } => {
-                // pop the None that was passed to kickstart the generator
-                self.emit(Instruction::Pop);
-            }
+            ast::ComprehensionKind::GeneratorExpression { .. } => {}
             ast::ComprehensionKind::List { .. } => {
                 self.emit(Instruction::BuildList {
                     size: 0,
diff --git a/src/ir.rs b/src/ir.rs
index dde8962a..177df64d 100644
--- a/src/ir.rs
+++ b/src/ir.rs
@@ -165,10 +165,7 @@ impl CodeInfo {
         let mut maxdepth = 0u32;
         let mut stack = Vec::with_capacity(self.blocks.len());
         let mut startdepths = vec![u32::MAX; self.blocks.len()];
-        // if it's a generator or a coroutine, it starts with something already on the stack
-        startdepths[0] =
-            self.flags
-                .intersects(CodeFlags::IS_GENERATOR | CodeFlags::IS_COROUTINE) as u32;
+        startdepths[0] = 0;
         stack.push(Label(0));
         'process_blocks: while let Some(block) = stack.pop() {
             let mut depth = startdepths[block.0 as usize];

From a0790d490606c4945a9edae9fc94792881c44fdc Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 19 Dec 2020 15:26:41 -0600
Subject: [PATCH 184/459] Use ahash in the compiler

---
 Cargo.toml         |  1 +
 src/compile.rs     | 14 +++++++-------
 src/ir.rs          |  2 +-
 src/lib.rs         |  3 +++
 src/symboltable.rs |  4 ++--
 5 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index ba37125e..7ceaed97 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -16,6 +16,7 @@ num-complex = { version = "0.3", features = ["serde"] }
 num-traits = "0.2"
 log = "0.4"
 arrayvec = "0.5"
+ahash = "0.6"
 
 [dev-dependencies]
 rustpython-parser = { path = "../parser" }
diff --git a/src/compile.rs b/src/compile.rs
index d9a1c3aa..46e0e4c1 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -9,7 +9,7 @@ use crate::error::{CompileError, CompileErrorType};
 use crate::ir::{self, CodeInfo};
 pub use crate::mode::Mode;
 use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolScope, SymbolTable};
-use indexmap::IndexSet;
+use crate::IndexSet;
 use itertools::Itertools;
 use num_complex::Complex64;
 use num_traits::ToPrimitive;
@@ -140,10 +140,10 @@ impl Compiler {
             blocks: vec![ir::Block::default()],
             current_block: bytecode::Label(0),
             constants: Vec::new(),
-            name_cache: IndexSet::new(),
-            varname_cache: IndexSet::new(),
-            cellvar_cache: IndexSet::new(),
-            freevar_cache: IndexSet::new(),
+            name_cache: IndexSet::default(),
+            varname_cache: IndexSet::default(),
+            cellvar_cache: IndexSet::default(),
+            freevar_cache: IndexSet::default(),
         };
         Compiler {
             code_stack: vec![module_code],
@@ -217,8 +217,8 @@ impl Compiler {
             blocks: vec![ir::Block::default()],
             current_block: bytecode::Label(0),
             constants: Vec::new(),
-            name_cache: IndexSet::new(),
-            varname_cache: IndexSet::new(),
+            name_cache: IndexSet::default(),
+            varname_cache: IndexSet::default(),
             cellvar_cache,
             freevar_cache,
         };
diff --git a/src/ir.rs b/src/ir.rs
index 177df64d..081eeb81 100644
--- a/src/ir.rs
+++ b/src/ir.rs
@@ -1,4 +1,4 @@
-use indexmap::IndexSet;
+use crate::IndexSet;
 use rustpython_bytecode::{CodeFlags, CodeObject, ConstantData, Instruction, Label, Location};
 
 pub type BlockIdx = Label;
diff --git a/src/lib.rs b/src/lib.rs
index 523f92e8..3116547d 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -5,6 +5,9 @@
 #[macro_use]
 extern crate log;
 
+type IndexMap<K, V> = indexmap::IndexMap<K, V, ahash::RandomState>;
+type IndexSet<T> = indexmap::IndexSet<T, ahash::RandomState>;
+
 pub mod compile;
 pub mod error;
 pub mod ir;
diff --git a/src/symboltable.rs b/src/symboltable.rs
index e5c238ea..4f22b8df 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -8,7 +8,7 @@ Inspirational file: https://github.com/python/cpython/blob/master/Python/symtabl
 */
 
 use crate::error::{CompileError, CompileErrorType};
-use indexmap::map::IndexMap;
+use crate::IndexMap;
 use rustpython_ast::{self as ast, Location};
 use std::fmt;
 
@@ -58,7 +58,7 @@ impl SymbolTable {
             typ,
             line_number,
             is_nested,
-            symbols: IndexMap::new(),
+            symbols: IndexMap::default(),
             sub_tables: vec![],
         }
     }

From 0e469eaf99e79e375f065410e013daab420d882c Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 9 Dec 2020 00:11:49 -0600
Subject: [PATCH 185/459] Use ahash/phf in parser

---
 Cargo.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 7ceaed97..9baa8ff4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,7 +15,6 @@ rustpython-ast = { path = "../ast" }
 num-complex = { version = "0.3", features = ["serde"] }
 num-traits = "0.2"
 log = "0.4"
-arrayvec = "0.5"
 ahash = "0.6"
 
 [dev-dependencies]

From 7c2c787404a2199436ac2722dadb6253b2256f77 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sat, 28 Nov 2020 14:57:00 -0600
Subject: [PATCH 186/459] Implement asdl in the ast and parser

---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 9baa8ff4..d302365e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -19,4 +19,4 @@ ahash = "0.6"
 
 [dev-dependencies]
 rustpython-parser = { path = "../parser" }
-insta = "1.1"
+insta = "1.5"

From dc8b93a4176f8f72454e617a56626ae416d75b18 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sun, 10 Jan 2021 00:46:59 -0600
Subject: [PATCH 187/459] Update the compiler to use the new asdl ast types

---
 porcelain/src/lib.rs |   35 +-
 src/compile.rs       | 1095 +++++++++++++++++++++---------------------
 src/error.rs         |    3 -
 src/symboltable.rs   |  362 +++++++-------
 4 files changed, 747 insertions(+), 748 deletions(-)

diff --git a/porcelain/src/lib.rs b/porcelain/src/lib.rs
index 9996c042..c4bd72a6 100644
--- a/porcelain/src/lib.rs
+++ b/porcelain/src/lib.rs
@@ -75,29 +75,16 @@ pub fn compile(
     source_path: String,
     opts: CompileOpts,
 ) -> Result<CodeObject, CompileError> {
-    macro_rules! try_parse {
-        ($x:expr) => {
-            match $x {
-                Ok(x) => x,
-                Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
-            }
-        };
-    }
-    let res = match mode {
-        compile::Mode::Exec => {
-            let ast = try_parse!(parser::parse_program(source));
-            compile::compile_program(ast, source_path, opts)
-        }
-        compile::Mode::Eval => {
-            let statement = try_parse!(parser::parse_statement(source));
-            compile::compile_statement_eval(statement, source_path, opts)
-        }
-        compile::Mode::Single => {
-            let ast = try_parse!(parser::parse_program(source));
-            compile::compile_program_single(ast, source_path, opts)
-        }
+    let mode = match mode {
+        compile::Mode::Exec => parser::Mode::Module,
+        compile::Mode::Eval => parser::Mode::Expression,
+        compile::Mode::Single => parser::Mode::Interactive,
+    };
+    let ast = match parser::parse(source, mode) {
+        Ok(x) => x,
+        Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
     };
-    res.map_err(|e| CompileError::from_compile(e, source))
+    compile::compile_top(&ast, source_path, opts).map_err(|e| CompileError::from_compile(e, source))
 }
 
 pub fn compile_symtable(
@@ -119,8 +106,8 @@ pub fn compile_symtable(
             symboltable::make_symbol_table(&ast)
         }
         compile::Mode::Eval => {
-            let statement = try_parse!(parser::parse_statement(source));
-            symboltable::statements_to_symbol_table(&statement)
+            let expr = try_parse!(parser::parse_expression(source));
+            symboltable::make_symbol_table_expr(&expr)
         }
     };
     res.map_err(|e| CompileError::from_symtable(e, source, source_path.to_owned()))
diff --git a/src/compile.rs b/src/compile.rs
index 46e0e4c1..bddf3d64 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -8,7 +8,7 @@
 use crate::error::{CompileError, CompileErrorType};
 use crate::ir::{self, CodeInfo};
 pub use crate::mode::Mode;
-use crate::symboltable::{make_symbol_table, statements_to_symbol_table, SymbolScope, SymbolTable};
+use crate::symboltable::{make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable};
 use crate::IndexSet;
 use itertools::Itertools;
 use num_complex::Complex64;
@@ -81,49 +81,62 @@ fn with_compiler(
     Ok(code)
 }
 
-/// Compile a standard Python program to bytecode
-pub fn compile_program(
-    ast: ast::Program,
+/// Compile an ast::Mod produced from rustpython_parser::parser::parse()
+pub fn compile_top(
+    ast: &ast::Mod,
     source_path: String,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    let symbol_table = match make_symbol_table(&ast) {
-        Ok(x) => x,
-        Err(e) => return Err(e.into_compile_error(source_path)),
-    };
-    with_compiler(source_path, opts, |compiler| {
-        compiler.compile_program(&ast, symbol_table)
-    })
+    match ast {
+        ast::Mod::Module { body, .. } => compile_program(body, source_path, opts),
+        ast::Mod::Interactive { body } => compile_program_single(body, source_path, opts),
+        ast::Mod::Expression { body } => compile_expression(body, source_path, opts),
+        ast::Mod::FunctionType { .. } => panic!("can't compile a FunctionType"),
+    }
+}
+
+macro_rules! compile_impl {
+    ($ast:expr, $source_path:expr, $opts:expr, $st:ident, $compile:ident) => {{
+        let symbol_table = match $st($ast) {
+            Ok(x) => x,
+            Err(e) => return Err(e.into_compile_error($source_path)),
+        };
+        with_compiler($source_path, $opts, |compiler| {
+            compiler.$compile($ast, symbol_table)
+        })
+    }};
 }
 
-/// Compile a single Python expression to bytecode
-pub fn compile_statement_eval(
-    statement: Vec<ast::Statement>,
+/// Compile a standard Python program to bytecode
+pub fn compile_program(
+    ast: &[ast::Stmt],
     source_path: String,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    let symbol_table = match statements_to_symbol_table(&statement) {
-        Ok(x) => x,
-        Err(e) => return Err(e.into_compile_error(source_path)),
-    };
-    with_compiler(source_path, opts, |compiler| {
-        compiler.compile_statement_eval(&statement, symbol_table)
-    })
+    compile_impl!(ast, source_path, opts, make_symbol_table, compile_program)
 }
 
 /// Compile a Python program to bytecode for the context of a REPL
 pub fn compile_program_single(
-    ast: ast::Program,
+    ast: &[ast::Stmt],
     source_path: String,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    let symbol_table = match make_symbol_table(&ast) {
-        Ok(x) => x,
-        Err(e) => return Err(e.into_compile_error(source_path)),
-    };
-    with_compiler(source_path, opts, |compiler| {
-        compiler.compile_program_single(&ast, symbol_table)
-    })
+    compile_impl!(
+        ast,
+        source_path,
+        opts,
+        make_symbol_table,
+        compile_program_single
+    )
+}
+
+pub fn compile_expression(
+    ast: &ast::Expr,
+    source_path: String,
+    opts: CompileOpts,
+) -> CompileResult<CodeObject> {
+    compile_impl!(ast, source_path, opts, make_symbol_table_expr, compile_eval)
 }
 
 impl Compiler {
@@ -255,13 +268,13 @@ impl Compiler {
 
     fn compile_program(
         &mut self,
-        program: &ast::Program,
+        body: &[ast::Stmt],
         symbol_table: SymbolTable,
     ) -> CompileResult<()> {
         let size_before = self.code_stack.len();
         self.symbol_table_stack.push(symbol_table);
 
-        let (statements, doc) = get_doc(&program.statements);
+        let (statements, doc) = get_doc(body);
         if let Some(value) = doc {
             self.emit_constant(ConstantData::Str { value });
             let doc = self.name("__doc__");
@@ -284,18 +297,18 @@ impl Compiler {
 
     fn compile_program_single(
         &mut self,
-        program: &ast::Program,
+        body: &[ast::Stmt],
         symbol_table: SymbolTable,
     ) -> CompileResult<()> {
         self.symbol_table_stack.push(symbol_table);
 
         let mut emitted_return = false;
 
-        for (i, statement) in program.statements.iter().enumerate() {
-            let is_last = i == program.statements.len() - 1;
+        for (i, statement) in body.iter().enumerate() {
+            let is_last = i == body.len() - 1;
 
-            if let ast::StatementType::Expression { ref expression } = statement.node {
-                self.compile_expression(expression)?;
+            if let ast::StmtKind::Expr { value } = &statement.node {
+                self.compile_expression(value)?;
 
                 if is_last {
                     self.emit(Instruction::Duplicate);
@@ -319,24 +332,18 @@ impl Compiler {
     }
 
     // Compile statement in eval mode:
-    fn compile_statement_eval(
+    fn compile_eval(
         &mut self,
-        statements: &[ast::Statement],
+        expression: &ast::Expr,
         symbol_table: SymbolTable,
     ) -> CompileResult<()> {
         self.symbol_table_stack.push(symbol_table);
-        for statement in statements {
-            if let ast::StatementType::Expression { ref expression } = statement.node {
-                self.compile_expression(expression)?;
-            } else {
-                return Err(self.error_loc(CompileErrorType::ExpectExpr, statement.location));
-            }
-        }
+        self.compile_expression(expression)?;
         self.emit(Instruction::ReturnValue);
         Ok(())
     }
 
-    fn compile_statements(&mut self, statements: &[ast::Statement]) -> CompileResult<()> {
+    fn compile_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> {
         for statement in statements {
             self.compile_statement(statement)?
         }
@@ -420,10 +427,10 @@ impl Compiler {
         self.emit(op(idx as u32));
     }
 
-    fn compile_statement(&mut self, statement: &ast::Statement) -> CompileResult<()> {
+    fn compile_statement(&mut self, statement: &ast::Stmt) -> CompileResult<()> {
         trace!("Compiling {:?}", statement);
         self.set_source_location(statement.location);
-        use ast::StatementType::*;
+        use ast::StmtKind::*;
 
         match &statement.node {
             // we do this here because `from __future__` still executes that `from` statement at runtime,
@@ -443,16 +450,16 @@ impl Compiler {
                         value: num_traits::Zero::zero(),
                     });
                     self.emit_constant(ConstantData::None);
-                    let idx = self.name(&name.symbol);
+                    let idx = self.name(&name.name);
                     self.emit(Instruction::ImportName { idx });
-                    if let Some(alias) = &name.alias {
-                        for part in name.symbol.split('.').skip(1) {
+                    if let Some(alias) = &name.asname {
+                        for part in name.name.split('.').skip(1) {
                             let idx = self.name(part);
                             self.emit(Instruction::LoadAttr { idx });
                         }
                         self.store_name(alias);
                     } else {
-                        self.store_name(name.symbol.split('.').next().unwrap());
+                        self.store_name(name.name.split('.').next().unwrap());
                     }
                 }
             }
@@ -461,7 +468,7 @@ impl Compiler {
                 module,
                 names,
             } => {
-                let import_star = names.iter().any(|n| n.symbol == "*");
+                let import_star = names.iter().any(|n| n.name == "*");
 
                 let from_list = if import_star {
                     if self.ctx.in_func() {
@@ -475,7 +482,7 @@ impl Compiler {
                     names
                         .iter()
                         .map(|n| ConstantData::Str {
-                            value: n.symbol.to_owned(),
+                            value: n.name.to_owned(),
                         })
                         .collect()
                 };
@@ -502,15 +509,15 @@ impl Compiler {
                     // from mod import a, b as c
 
                     for name in names {
-                        let idx = self.name(&name.symbol);
+                        let idx = self.name(&name.name);
                         // import symbol from module:
                         self.emit(Instruction::ImportFrom { idx });
 
                         // Store module under proper name:
-                        if let Some(alias) = &name.alias {
+                        if let Some(alias) = &name.asname {
                             self.store_name(alias);
                         } else {
-                            self.store_name(&name.symbol);
+                            self.store_name(&name.name);
                         }
                     }
 
@@ -518,8 +525,8 @@ impl Compiler {
                     self.emit(Instruction::Pop);
                 }
             }
-            Expression { expression } => {
-                self.compile_expression(expression)?;
+            Expr { value } => {
+                self.compile_expression(value)?;
 
                 // Pop result of stack, since we not use it:
                 self.emit(Instruction::Pop);
@@ -529,93 +536,44 @@ impl Compiler {
             }
             If { test, body, orelse } => {
                 let after_block = self.new_block();
-                match orelse {
-                    None => {
-                        // Only if:
-                        self.compile_jump_if(test, false, after_block)?;
-                        self.compile_statements(body)?;
-                    }
-                    Some(statements) => {
-                        // if - else:
-                        let else_block = self.new_block();
-                        self.compile_jump_if(test, false, else_block)?;
-                        self.compile_statements(body)?;
-                        self.emit(Instruction::Jump {
-                            target: after_block,
-                        });
+                if orelse.is_empty() {
+                    // Only if:
+                    self.compile_jump_if(test, false, after_block)?;
+                    self.compile_statements(body)?;
+                } else {
+                    // if - else:
+                    let else_block = self.new_block();
+                    self.compile_jump_if(test, false, else_block)?;
+                    self.compile_statements(body)?;
+                    self.emit(Instruction::Jump {
+                        target: after_block,
+                    });
 
-                        // else:
-                        self.switch_to_block(else_block);
-                        self.compile_statements(statements)?;
-                    }
+                    // else:
+                    self.switch_to_block(else_block);
+                    self.compile_statements(orelse)?;
                 }
                 self.switch_to_block(after_block);
             }
             While { test, body, orelse } => self.compile_while(test, body, orelse)?,
-            With {
-                is_async,
-                items,
-                body,
-            } => {
-                let is_async = *is_async;
-
-                let end_blocks = items
-                    .iter()
-                    .map(|item| {
-                        let end_block = self.new_block();
-                        self.compile_expression(&item.context_expr)?;
-
-                        if is_async {
-                            self.emit(Instruction::BeforeAsyncWith);
-                            self.emit(Instruction::GetAwaitable);
-                            self.emit_constant(ConstantData::None);
-                            self.emit(Instruction::YieldFrom);
-                            self.emit(Instruction::SetupAsyncWith { end: end_block });
-                        } else {
-                            self.emit(Instruction::SetupWith { end: end_block });
-                        }
-
-                        match &item.optional_vars {
-                            Some(var) => {
-                                self.compile_store(var)?;
-                            }
-                            None => {
-                                self.emit(Instruction::Pop);
-                            }
-                        }
-                        Ok(end_block)
-                    })
-                    .collect::<CompileResult<Vec<_>>>()?;
-
-                self.compile_statements(body)?;
-
-                // sort of "stack up" the layers of with blocks:
-                // with a, b: body -> start_with(a) start_with(b) body() end_with(b) end_with(a)
-                for end_block in end_blocks.into_iter().rev() {
-                    self.emit(Instruction::PopBlock);
-                    self.emit(Instruction::EnterFinally);
-
-                    self.switch_to_block(end_block);
-                    self.emit(Instruction::WithCleanupStart);
-
-                    if is_async {
-                        self.emit(Instruction::GetAwaitable);
-                        self.emit_constant(ConstantData::None);
-                        self.emit(Instruction::YieldFrom);
-                    }
-
-                    self.emit(Instruction::WithCleanupFinish);
-                }
-            }
+            With { items, body, .. } => self.compile_with(items, body, false)?,
+            AsyncWith { items, body, .. } => self.compile_with(items, body, true)?,
             For {
-                is_async,
                 target,
                 iter,
                 body,
                 orelse,
-            } => self.compile_for(target, iter, body, orelse, *is_async)?,
-            Raise { exception, cause } => {
-                let kind = match exception {
+                ..
+            } => self.compile_for(target, iter, body, orelse, false)?,
+            AsyncFor {
+                target,
+                iter,
+                body,
+                orelse,
+                ..
+            } => self.compile_for(target, iter, body, orelse, true)?,
+            Raise { exc, cause } => {
+                let kind = match exc {
                     Some(value) => {
                         self.compile_expression(value)?;
                         match cause {
@@ -637,15 +595,35 @@ impl Compiler {
                 finalbody,
             } => self.compile_try_statement(body, handlers, orelse, finalbody)?,
             FunctionDef {
-                is_async,
                 name,
                 args,
                 body,
                 decorator_list,
                 returns,
-            } => {
-                self.compile_function_def(name, args, body, decorator_list, returns, *is_async)?;
-            }
+                ..
+            } => self.compile_function_def(
+                name,
+                args,
+                body,
+                decorator_list,
+                returns.as_deref(),
+                false,
+            )?,
+            AsyncFunctionDef {
+                name,
+                args,
+                body,
+                decorator_list,
+                returns,
+                ..
+            } => self.compile_function_def(
+                name,
+                args,
+                body,
+                decorator_list,
+                returns.as_deref(),
+                true,
+            )?,
             ClassDef {
                 name,
                 body,
@@ -721,7 +699,7 @@ impl Compiler {
 
                 self.emit(Instruction::ReturnValue);
             }
-            Assign { targets, value } => {
+            Assign { targets, value, .. } => {
                 self.compile_expression(value)?;
 
                 for (i, target) in targets.iter().enumerate() {
@@ -743,7 +721,8 @@ impl Compiler {
                 target,
                 annotation,
                 value,
-            } => self.compile_annotated_assign(target, annotation, value)?,
+                ..
+            } => self.compile_annotated_assign(target, annotation, value.as_deref())?,
             Delete { targets } => {
                 for target in targets {
                     self.compile_delete(target)?;
@@ -756,27 +735,27 @@ impl Compiler {
         Ok(())
     }
 
-    fn compile_delete(&mut self, expression: &ast::Expression) -> CompileResult<()> {
+    fn compile_delete(&mut self, expression: &ast::Expr) -> CompileResult<()> {
         match &expression.node {
-            ast::ExpressionType::Identifier { name } => {
-                self.compile_name(name, NameUsage::Delete);
+            ast::ExprKind::Name { id, .. } => {
+                self.compile_name(id, NameUsage::Delete);
             }
-            ast::ExpressionType::Attribute { value, name } => {
+            ast::ExprKind::Attribute { value, attr, .. } => {
                 self.compile_expression(value)?;
-                let idx = self.name(name);
+                let idx = self.name(attr);
                 self.emit(Instruction::DeleteAttr { idx });
             }
-            ast::ExpressionType::Subscript { a, b } => {
-                self.compile_expression(a)?;
-                self.compile_expression(b)?;
+            ast::ExprKind::Subscript { value, slice, .. } => {
+                self.compile_expression(value)?;
+                self.compile_expression(slice)?;
                 self.emit(Instruction::DeleteSubscript);
             }
-            ast::ExpressionType::Tuple { elements } => {
-                for element in elements {
+            ast::ExprKind::Tuple { elts, .. } => {
+                for element in elts {
                     self.compile_delete(element)?;
                 }
             }
-            _ => return Err(self.error(CompileErrorType::Delete(expression.name()))),
+            _ => return Err(self.error(CompileErrorType::Delete(expression.node.name()))),
         }
         Ok(())
     }
@@ -784,7 +763,7 @@ impl Compiler {
     fn enter_function(
         &mut self,
         name: &str,
-        args: &ast::Parameters,
+        args: &ast::Arguments,
     ) -> CompileResult<bytecode::MakeFunctionFlags> {
         let have_defaults = !args.defaults.is_empty();
         if have_defaults {
@@ -803,7 +782,7 @@ impl Compiler {
         for (kw, default) in args.kwonlyargs.iter().zip(&args.kw_defaults) {
             if let Some(default) = default {
                 self.emit_constant(ConstantData::Str {
-                    value: kw.arg.clone(),
+                    value: kw.node.arg.clone(),
                 });
                 self.compile_expression(default)?;
                 num_kw_only_defaults += 1;
@@ -827,41 +806,41 @@ impl Compiler {
 
         self.push_output(
             bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED,
-            args.posonlyargs_count,
-            args.args.len(),
+            args.posonlyargs.len(),
+            args.posonlyargs.len() + args.args.len(),
             args.kwonlyargs.len(),
             name.to_owned(),
         );
 
-        for name in &args.args {
-            self.varname(&name.arg);
-        }
-        for name in &args.kwonlyargs {
-            self.varname(&name.arg);
+        let args_iter = std::iter::empty()
+            .chain(&args.posonlyargs)
+            .chain(&args.args)
+            .chain(&args.kwonlyargs);
+        for name in args_iter {
+            self.varname(&name.node.arg);
         }
 
-        let mut compile_varargs = |va: &ast::Varargs, flag| match va {
-            ast::Varargs::None | ast::Varargs::Unnamed => {}
-            ast::Varargs::Named(name) => {
+        let mut compile_varargs = |va: Option<&ast::Arg>, flag| {
+            if let Some(name) = va {
                 self.current_codeinfo().flags |= flag;
-                self.varname(&name.arg);
+                self.varname(&name.node.arg);
             }
         };
 
-        compile_varargs(&args.vararg, bytecode::CodeFlags::HAS_VARARGS);
-        compile_varargs(&args.kwarg, bytecode::CodeFlags::HAS_VARKEYWORDS);
+        compile_varargs(args.vararg.as_deref(), bytecode::CodeFlags::HAS_VARARGS);
+        compile_varargs(args.kwarg.as_deref(), bytecode::CodeFlags::HAS_VARKEYWORDS);
 
         Ok(funcflags)
     }
 
-    fn prepare_decorators(&mut self, decorator_list: &[ast::Expression]) -> CompileResult<()> {
+    fn prepare_decorators(&mut self, decorator_list: &[ast::Expr]) -> CompileResult<()> {
         for decorator in decorator_list {
             self.compile_expression(decorator)?;
         }
         Ok(())
     }
 
-    fn apply_decorators(&mut self, decorator_list: &[ast::Expression]) {
+    fn apply_decorators(&mut self, decorator_list: &[ast::Expr]) {
         // Apply decorators:
         for _ in decorator_list {
             self.emit(Instruction::CallFunctionPositional { nargs: 1 });
@@ -870,16 +849,16 @@ impl Compiler {
 
     fn compile_try_statement(
         &mut self,
-        body: &[ast::Statement],
-        handlers: &[ast::ExceptHandler],
-        orelse: &Option<ast::Suite>,
-        finalbody: &Option<ast::Suite>,
+        body: &[ast::Stmt],
+        handlers: &[ast::Excepthandler],
+        orelse: &[ast::Stmt],
+        finalbody: &[ast::Stmt],
     ) -> CompileResult<()> {
         let handler_block = self.new_block();
         let finally_block = self.new_block();
 
         // Setup a finally block if we have a finally statement.
-        if finalbody.is_some() {
+        if !finalbody.is_empty() {
             self.emit(Instruction::SetupFinally {
                 handler: finally_block,
             });
@@ -899,11 +878,12 @@ impl Compiler {
         self.switch_to_block(handler_block);
         // Exception is on top of stack now
         for handler in handlers {
+            let ast::ExcepthandlerKind::ExceptHandler { type_, name, body } = &handler.node;
             let next_handler = self.new_block();
 
             // If we gave a typ,
             // check if this handler can handle the exception:
-            if let Some(exc_type) = &handler.typ {
+            if let Some(exc_type) = type_ {
                 // Duplicate exception for test:
                 self.emit(Instruction::Duplicate);
 
@@ -919,7 +899,7 @@ impl Compiler {
                 });
 
                 // We have a match, store in name (except x as y)
-                if let Some(alias) = &handler.name {
+                if let Some(alias) = name {
                     self.store_name(alias);
                 } else {
                     // Drop exception from top of stack:
@@ -932,10 +912,10 @@ impl Compiler {
             }
 
             // Handler code:
-            self.compile_statements(&handler.body)?;
+            self.compile_statements(body)?;
             self.emit(Instruction::PopException);
 
-            if finalbody.is_some() {
+            if !finalbody.is_empty() {
                 self.emit(Instruction::PopBlock); // pop excepthandler block
                                                   // We enter the finally block, without exception.
                 self.emit(Instruction::EnterFinally);
@@ -958,11 +938,9 @@ impl Compiler {
         // We successfully ran the try block:
         // else:
         self.switch_to_block(else_block);
-        if let Some(statements) = orelse {
-            self.compile_statements(statements)?;
-        }
+        self.compile_statements(orelse)?;
 
-        if finalbody.is_some() {
+        if !finalbody.is_empty() {
             self.emit(Instruction::PopBlock); // pop finally block
 
             // We enter the finallyhandler block, without return / exception.
@@ -971,8 +949,8 @@ impl Compiler {
 
         // finally:
         self.switch_to_block(finally_block);
-        if let Some(statements) = finalbody {
-            self.compile_statements(statements)?;
+        if !finalbody.is_empty() {
+            self.compile_statements(finalbody)?;
             self.emit(Instruction::EndFinally);
         }
 
@@ -982,10 +960,10 @@ impl Compiler {
     fn compile_function_def(
         &mut self,
         name: &str,
-        args: &ast::Parameters,
-        body: &[ast::Statement],
-        decorator_list: &[ast::Expression],
-        returns: &Option<ast::Expression>, // TODO: use type hint somehow..
+        args: &ast::Arguments,
+        body: &[ast::Stmt],
+        decorator_list: &[ast::Expr],
+        returns: Option<&ast::Expr>, // TODO: use type hint somehow..
         is_async: bool,
     ) -> CompileResult<()> {
         // Create bytecode for this function:
@@ -1019,7 +997,7 @@ impl Compiler {
 
         // Emit None at end:
         match body.last().map(|s| &s.node) {
-            Some(ast::StatementType::Return { .. }) => {
+            Some(ast::StmtKind::Return { .. }) => {
                 // the last instruction is a ReturnValue already, we don't need to emit it
             }
             _ => {
@@ -1046,27 +1024,20 @@ impl Compiler {
             num_annotations += 1;
         }
 
-        let mut visit_arg_annotation = |arg: &ast::Parameter| -> CompileResult<()> {
-            if let Some(annotation) = &arg.annotation {
+        let args_iter = std::iter::empty()
+            .chain(&args.posonlyargs)
+            .chain(&args.args)
+            .chain(&args.kwonlyargs)
+            .chain(args.vararg.as_deref())
+            .chain(args.kwarg.as_deref());
+        for arg in args_iter {
+            if let Some(annotation) = &arg.node.annotation {
                 self.emit_constant(ConstantData::Str {
-                    value: arg.arg.to_owned(),
+                    value: arg.node.arg.to_owned(),
                 });
                 self.compile_expression(&annotation)?;
                 num_annotations += 1;
             }
-            Ok(())
-        };
-
-        for arg in args.args.iter().chain(args.kwonlyargs.iter()) {
-            visit_arg_annotation(arg)?;
-        }
-
-        if let ast::Varargs::Named(arg) = &args.vararg {
-            visit_arg_annotation(arg)?;
-        }
-
-        if let ast::Varargs::Named(arg) = &args.kwarg {
-            visit_arg_annotation(arg)?;
         }
 
         if num_annotations > 0 {
@@ -1135,54 +1106,22 @@ impl Compiler {
         true
     }
 
-    fn find_ann(&self, body: &[ast::Statement]) -> bool {
-        use ast::StatementType::*;
-        let option_stmt_to_bool = |suit: &Option<ast::Suite>| -> bool {
-            match suit {
-                Some(stmts) => self.find_ann(stmts),
-                None => false,
-            }
-        };
+    fn find_ann(&self, body: &[ast::Stmt]) -> bool {
+        use ast::StmtKind::*;
 
         for statement in body {
             let res = match &statement.node {
-                AnnAssign {
-                    target: _,
-                    annotation: _,
-                    value: _,
-                } => true,
-                For {
-                    is_async: _,
-                    target: _,
-                    iter: _,
-                    body,
-                    orelse,
-                } => self.find_ann(body) || option_stmt_to_bool(orelse),
-                If {
-                    test: _,
-                    body,
-                    orelse,
-                } => self.find_ann(body) || option_stmt_to_bool(orelse),
-                While {
-                    test: _,
-                    body,
-                    orelse,
-                } => self.find_ann(body) || option_stmt_to_bool(orelse),
-                With {
-                    is_async: _,
-                    items: _,
-                    body,
-                } => self.find_ann(body),
+                AnnAssign { .. } => true,
+                For { body, orelse, .. } => self.find_ann(body) || self.find_ann(orelse),
+                If { body, orelse, .. } => self.find_ann(body) || self.find_ann(orelse),
+                While { body, orelse, .. } => self.find_ann(body) || self.find_ann(orelse),
+                With { body, .. } => self.find_ann(body),
                 Try {
                     body,
-                    handlers: _,
                     orelse,
                     finalbody,
-                } => {
-                    self.find_ann(&body)
-                        || option_stmt_to_bool(orelse)
-                        || option_stmt_to_bool(finalbody)
-                }
+                    ..
+                } => self.find_ann(&body) || self.find_ann(orelse) || self.find_ann(finalbody),
                 _ => false,
             };
             if res {
@@ -1195,10 +1134,10 @@ impl Compiler {
     fn compile_class_def(
         &mut self,
         name: &str,
-        body: &[ast::Statement],
-        bases: &[ast::Expression],
+        body: &[ast::Stmt],
+        bases: &[ast::Expr],
         keywords: &[ast::Keyword],
-        decorator_list: &[ast::Expression],
+        decorator_list: &[ast::Expr],
     ) -> CompileResult<()> {
         self.prepare_decorators(decorator_list)?;
 
@@ -1287,7 +1226,7 @@ impl Compiler {
         if !keywords.is_empty() {
             let mut kwarg_names = vec![];
             for keyword in keywords {
-                if let Some(name) = &keyword.name {
+                if let Some(name) = &keyword.node.arg {
                     kwarg_names.push(ConstantData::Str {
                         value: name.to_owned(),
                     });
@@ -1295,7 +1234,7 @@ impl Compiler {
                     // This means **kwargs!
                     panic!("name must be set");
                 }
-                self.compile_expression(&keyword.value)?;
+                self.compile_expression(&keyword.node.value)?;
             }
 
             self.emit_constant(ConstantData::Tuple {
@@ -1329,9 +1268,9 @@ impl Compiler {
 
     fn compile_while(
         &mut self,
-        test: &ast::Expression,
-        body: &[ast::Statement],
-        orelse: &Option<Vec<ast::Statement>>,
+        test: &ast::Expr,
+        body: &[ast::Stmt],
+        orelse: &[ast::Stmt],
     ) -> CompileResult<()> {
         let while_block = self.new_block();
         let else_block = self.new_block();
@@ -1351,19 +1290,74 @@ impl Compiler {
         });
         self.switch_to_block(else_block);
         self.emit(Instruction::PopBlock);
-        if let Some(orelse) = orelse {
-            self.compile_statements(orelse)?;
-        }
+        self.compile_statements(orelse)?;
         self.switch_to_block(after_block);
         Ok(())
     }
 
+    fn compile_with(
+        &mut self,
+        items: &[ast::Withitem],
+        body: &[ast::Stmt],
+        is_async: bool,
+    ) -> CompileResult<()> {
+        let end_blocks = items
+            .iter()
+            .map(|item| {
+                let end_block = self.new_block();
+                self.compile_expression(&item.context_expr)?;
+
+                if is_async {
+                    self.emit(Instruction::BeforeAsyncWith);
+                    self.emit(Instruction::GetAwaitable);
+                    self.emit_constant(ConstantData::None);
+                    self.emit(Instruction::YieldFrom);
+                    self.emit(Instruction::SetupAsyncWith { end: end_block });
+                } else {
+                    self.emit(Instruction::SetupWith { end: end_block });
+                }
+
+                match &item.optional_vars {
+                    Some(var) => {
+                        self.compile_store(var)?;
+                    }
+                    None => {
+                        self.emit(Instruction::Pop);
+                    }
+                }
+                Ok(end_block)
+            })
+            .collect::<CompileResult<Vec<_>>>()?;
+
+        self.compile_statements(body)?;
+
+        // sort of "stack up" the layers of with blocks:
+        // with a, b: body -> start_with(a) start_with(b) body() end_with(b) end_with(a)
+        for end_block in end_blocks.into_iter().rev() {
+            self.emit(Instruction::PopBlock);
+            self.emit(Instruction::EnterFinally);
+
+            self.switch_to_block(end_block);
+            self.emit(Instruction::WithCleanupStart);
+
+            if is_async {
+                self.emit(Instruction::GetAwaitable);
+                self.emit_constant(ConstantData::None);
+                self.emit(Instruction::YieldFrom);
+            }
+
+            self.emit(Instruction::WithCleanupFinish);
+        }
+
+        Ok(())
+    }
+
     fn compile_for(
         &mut self,
-        target: &ast::Expression,
-        iter: &ast::Expression,
-        body: &[ast::Statement],
-        orelse: &Option<Vec<ast::Statement>>,
+        target: &ast::Expr,
+        iter: &ast::Expr,
+        body: &[ast::Stmt],
+        orelse: &[ast::Stmt],
         is_async: bool,
     ) -> CompileResult<()> {
         // Start loop
@@ -1417,9 +1411,7 @@ impl Compiler {
 
         self.switch_to_block(else_block);
         self.emit(Instruction::PopBlock);
-        if let Some(orelse) = orelse {
-            self.compile_statements(orelse)?;
-        }
+        self.compile_statements(orelse)?;
 
         self.switch_to_block(after_block);
 
@@ -1428,23 +1420,26 @@ impl Compiler {
 
     fn compile_chained_comparison(
         &mut self,
-        vals: &[ast::Expression],
-        ops: &[ast::Comparison],
+        left: &ast::Expr,
+        ops: &[ast::Cmpop],
+        vals: &[ast::Expr],
     ) -> CompileResult<()> {
         assert!(!ops.is_empty());
-        assert_eq!(vals.len(), ops.len() + 1);
-
-        let to_operator = |op: &ast::Comparison| match op {
-            ast::Comparison::Equal => bytecode::ComparisonOperator::Equal,
-            ast::Comparison::NotEqual => bytecode::ComparisonOperator::NotEqual,
-            ast::Comparison::Less => bytecode::ComparisonOperator::Less,
-            ast::Comparison::LessOrEqual => bytecode::ComparisonOperator::LessOrEqual,
-            ast::Comparison::Greater => bytecode::ComparisonOperator::Greater,
-            ast::Comparison::GreaterOrEqual => bytecode::ComparisonOperator::GreaterOrEqual,
-            ast::Comparison::In => bytecode::ComparisonOperator::In,
-            ast::Comparison::NotIn => bytecode::ComparisonOperator::NotIn,
-            ast::Comparison::Is => bytecode::ComparisonOperator::Is,
-            ast::Comparison::IsNot => bytecode::ComparisonOperator::IsNot,
+        assert_eq!(vals.len(), ops.len());
+        let (last_op, mid_ops) = ops.split_last().unwrap();
+        let (last_val, mid_vals) = vals.split_last().unwrap();
+
+        let compile_cmpop = |op: &ast::Cmpop| match op {
+            ast::Cmpop::Eq => bytecode::ComparisonOperator::Equal,
+            ast::Cmpop::NotEq => bytecode::ComparisonOperator::NotEqual,
+            ast::Cmpop::Lt => bytecode::ComparisonOperator::Less,
+            ast::Cmpop::LtE => bytecode::ComparisonOperator::LessOrEqual,
+            ast::Cmpop::Gt => bytecode::ComparisonOperator::Greater,
+            ast::Cmpop::GtE => bytecode::ComparisonOperator::GreaterOrEqual,
+            ast::Cmpop::In => bytecode::ComparisonOperator::In,
+            ast::Cmpop::NotIn => bytecode::ComparisonOperator::NotIn,
+            ast::Cmpop::Is => bytecode::ComparisonOperator::Is,
+            ast::Cmpop::IsNot => bytecode::ComparisonOperator::IsNot,
         };
 
         // a == b == c == d
@@ -1456,27 +1451,25 @@ impl Compiler {
         //     result = c == d
 
         // initialize lhs outside of loop
-        self.compile_expression(&vals[0])?;
+        self.compile_expression(left)?;
 
-        let end_blocks = if vals.len() > 2 {
+        let end_blocks = if mid_vals.is_empty() {
+            None
+        } else {
             let break_block = self.new_block();
             let after_block = self.new_block();
             Some((break_block, after_block))
-        } else {
-            None
         };
 
         // for all comparisons except the last (as the last one doesn't need a conditional jump)
-        let ops_slice = &ops[0..ops.len()];
-        let vals_slice = &vals[1..ops.len()];
-        for (op, val) in ops_slice.iter().zip(vals_slice.iter()) {
+        for (op, val) in mid_ops.iter().zip(mid_vals) {
             self.compile_expression(val)?;
             // store rhs for the next comparison in chain
             self.emit(Instruction::Duplicate);
             self.emit(Instruction::Rotate { amount: 3 });
 
             self.emit(Instruction::CompareOperation {
-                op: to_operator(op),
+                op: compile_cmpop(op),
             });
 
             // if comparison result is false, we break with this value; if true, try the next one.
@@ -1488,9 +1481,9 @@ impl Compiler {
         }
 
         // handle the last comparison
-        self.compile_expression(vals.last().unwrap())?;
+        self.compile_expression(last_val)?;
         self.emit(Instruction::CompareOperation {
-            op: to_operator(ops.last().unwrap()),
+            op: compile_cmpop(last_op),
         });
 
         if let Some((break_block, after_block)) = end_blocks {
@@ -1511,9 +1504,9 @@ impl Compiler {
 
     fn compile_annotated_assign(
         &mut self,
-        target: &ast::Expression,
-        annotation: &ast::Expression,
-        value: &Option<ast::Expression>,
+        target: &ast::Expr,
+        annotation: &ast::Expr,
+        value: Option<&ast::Expr>,
     ) -> CompileResult<()> {
         if let Some(value) = value {
             self.compile_expression(value)?;
@@ -1528,12 +1521,12 @@ impl Compiler {
         // Compile annotation:
         self.compile_expression(annotation)?;
 
-        if let ast::ExpressionType::Identifier { name } = &target.node {
+        if let ast::ExprKind::Name { id, .. } = &target.node {
             // Store as dict entry in __annotations__ dict:
             let annotations = self.name("__annotations__");
             self.emit(Instruction::LoadNameAny(annotations));
             self.emit_constant(ConstantData::Str {
-                value: name.to_owned(),
+                value: id.to_owned(),
             });
             self.emit(Instruction::StoreSubscript);
         } else {
@@ -1544,33 +1537,33 @@ impl Compiler {
         Ok(())
     }
 
-    fn compile_store(&mut self, target: &ast::Expression) -> CompileResult<()> {
+    fn compile_store(&mut self, target: &ast::Expr) -> CompileResult<()> {
         match &target.node {
-            ast::ExpressionType::Identifier { name } => {
-                self.store_name(name);
+            ast::ExprKind::Name { id, .. } => {
+                self.store_name(id);
             }
-            ast::ExpressionType::Subscript { a, b } => {
-                self.compile_expression(a)?;
-                self.compile_expression(b)?;
+            ast::ExprKind::Subscript { value, slice, .. } => {
+                self.compile_expression(value)?;
+                self.compile_expression(slice)?;
                 self.emit(Instruction::StoreSubscript);
             }
-            ast::ExpressionType::Attribute { value, name } => {
+            ast::ExprKind::Attribute { value, attr, .. } => {
                 self.compile_expression(value)?;
-                let idx = self.name(name);
+                let idx = self.name(attr);
                 self.emit(Instruction::StoreAttr { idx });
             }
-            ast::ExpressionType::List { elements } | ast::ExpressionType::Tuple { elements } => {
+            ast::ExprKind::List { elts, .. } | ast::ExprKind::Tuple { elts, .. } => {
                 let mut seen_star = false;
 
                 // Scan for star args:
-                for (i, element) in elements.iter().enumerate() {
-                    if let ast::ExpressionType::Starred { .. } = &element.node {
+                for (i, element) in elts.iter().enumerate() {
+                    if let ast::ExprKind::Starred { .. } = &element.node {
                         if seen_star {
                             return Err(self.error(CompileErrorType::MultipleStarArgs));
                         } else {
                             seen_star = true;
                             let before = i;
-                            let after = elements.len() - i - 1;
+                            let after = elts.len() - i - 1;
                             let (before, after) = (|| Some((before.to_u8()?, after.to_u8()?)))()
                                 .ok_or_else(|| {
                                     self.error_loc(
@@ -1585,12 +1578,12 @@ impl Compiler {
 
                 if !seen_star {
                     self.emit(Instruction::UnpackSequence {
-                        size: elements.len() as u32,
+                        size: elts.len() as u32,
                     });
                 }
 
-                for element in elements {
-                    if let ast::ExpressionType::Starred { value } = &element.node {
+                for element in elts {
+                    if let ast::ExprKind::Starred { value, .. } = &element.node {
                         self.compile_store(value)?;
                     } else {
                         self.compile_store(element)?;
@@ -1599,10 +1592,10 @@ impl Compiler {
             }
             _ => {
                 return Err(self.error(match target.node {
-                    ast::ExpressionType::Starred { .. } => CompileErrorType::SyntaxError(
+                    ast::ExprKind::Starred { .. } => CompileErrorType::SyntaxError(
                         "starred assignment target must be in a list or tuple".to_owned(),
                     ),
-                    _ => CompileErrorType::Assign(target.name()),
+                    _ => CompileErrorType::Assign(target.node.name()),
                 }))
             }
         }
@@ -1644,15 +1637,15 @@ impl Compiler {
     /// (indicated by the condition parameter).
     fn compile_jump_if(
         &mut self,
-        expression: &ast::Expression,
+        expression: &ast::Expr,
         condition: bool,
         target_block: ir::BlockIdx,
     ) -> CompileResult<()> {
         // Compile expression for test, and jump to label if false
         match &expression.node {
-            ast::ExpressionType::BoolOp { op, values } => {
+            ast::ExprKind::BoolOp { op, values } => {
                 match op {
-                    ast::BooleanOperator::And => {
+                    ast::Boolop::And => {
                         if condition {
                             // If all values are true.
                             let end_block = self.new_block();
@@ -1673,7 +1666,7 @@ impl Compiler {
                             }
                         }
                     }
-                    ast::BooleanOperator::Or => {
+                    ast::Boolop::Or => {
                         if condition {
                             // If any of the values is true.
                             for value in values {
@@ -1696,11 +1689,11 @@ impl Compiler {
                     }
                 }
             }
-            ast::ExpressionType::Unop {
-                op: ast::UnaryOperator::Not,
-                a,
+            ast::ExprKind::UnaryOp {
+                op: ast::Unaryop::Not,
+                operand,
             } => {
-                self.compile_jump_if(a, !condition, target_block)?;
+                self.compile_jump_if(operand, !condition, target_block)?;
             }
             _ => {
                 // Fall back case which always will work!
@@ -1721,11 +1714,7 @@ impl Compiler {
 
     /// Compile a boolean operation as an expression.
     /// This means, that the last value remains on the stack.
-    fn compile_bool_op(
-        &mut self,
-        op: &ast::BooleanOperator,
-        values: &[ast::Expression],
-    ) -> CompileResult<()> {
+    fn compile_bool_op(&mut self, op: &ast::Boolop, values: &[ast::Expr]) -> CompileResult<()> {
         let after_block = self.new_block();
 
         let (last_value, values) = values.split_last().unwrap();
@@ -1733,12 +1722,12 @@ impl Compiler {
             self.compile_expression(value)?;
 
             match op {
-                ast::BooleanOperator::And => {
+                ast::Boolop::And => {
                     self.emit(Instruction::JumpIfFalseOrPop {
                         target: after_block,
                     });
                 }
-                ast::BooleanOperator::Or => {
+                ast::Boolop::Or => {
                     self.emit(Instruction::JumpIfTrueOrPop {
                         target: after_block,
                     });
@@ -1754,11 +1743,12 @@ impl Compiler {
 
     fn compile_dict(
         &mut self,
-        pairs: &[(Option<ast::Expression>, ast::Expression)],
+        keys: &[Option<Box<ast::Expr>>],
+        values: &[ast::Expr],
     ) -> CompileResult<()> {
         let mut size = 0;
         let mut has_unpacking = false;
-        for (is_unpacking, subpairs) in &pairs.iter().group_by(|e| e.0.is_none()) {
+        for (is_unpacking, subpairs) in &keys.iter().zip(values).group_by(|e| e.0.is_none()) {
             if is_unpacking {
                 for (_, value) in subpairs {
                     self.compile_expression(value)?;
@@ -1799,95 +1789,99 @@ impl Compiler {
         Ok(())
     }
 
-    fn compile_expression(&mut self, expression: &ast::Expression) -> CompileResult<()> {
+    fn compile_expression(&mut self, expression: &ast::Expr) -> CompileResult<()> {
         trace!("Compiling {:?}", expression);
         self.set_source_location(expression.location);
 
-        use ast::ExpressionType::*;
+        use ast::ExprKind::*;
         match &expression.node {
             Call {
-                function,
+                func,
                 args,
                 keywords,
-            } => self.compile_call(function, args, keywords)?,
+            } => self.compile_call(func, args, keywords)?,
             BoolOp { op, values } => self.compile_bool_op(op, values)?,
-            Binop { a, op, b } => {
-                self.compile_expression(a)?;
-                self.compile_expression(b)?;
+            BinOp { left, op, right } => {
+                self.compile_expression(left)?;
+                self.compile_expression(right)?;
 
                 // Perform operation:
                 self.compile_op(op, false);
             }
-            Subscript { a, b } => {
-                self.compile_expression(a)?;
-                self.compile_expression(b)?;
+            Subscript { value, slice, .. } => {
+                self.compile_expression(value)?;
+                self.compile_expression(slice)?;
                 self.emit(Instruction::Subscript);
             }
-            Unop { op, a } => {
-                self.compile_expression(a)?;
+            UnaryOp { op, operand } => {
+                self.compile_expression(operand)?;
 
                 // Perform operation:
                 let i = match op {
-                    ast::UnaryOperator::Pos => bytecode::UnaryOperator::Plus,
-                    ast::UnaryOperator::Neg => bytecode::UnaryOperator::Minus,
-                    ast::UnaryOperator::Not => bytecode::UnaryOperator::Not,
-                    ast::UnaryOperator::Inv => bytecode::UnaryOperator::Invert,
+                    ast::Unaryop::UAdd => bytecode::UnaryOperator::Plus,
+                    ast::Unaryop::USub => bytecode::UnaryOperator::Minus,
+                    ast::Unaryop::Not => bytecode::UnaryOperator::Not,
+                    ast::Unaryop::Invert => bytecode::UnaryOperator::Invert,
                 };
                 let i = Instruction::UnaryOperation { op: i };
                 self.emit(i);
             }
-            Attribute { value, name } => {
+            Attribute { value, attr, .. } => {
                 self.compile_expression(value)?;
-                let idx = self.name(name);
+                let idx = self.name(attr);
                 self.emit(Instruction::LoadAttr { idx });
             }
-            Compare { vals, ops } => {
-                self.compile_chained_comparison(vals, ops)?;
+            Compare {
+                left,
+                ops,
+                comparators,
+            } => {
+                self.compile_chained_comparison(left, ops, comparators)?;
             }
-            Number { value } => {
-                let const_value = match value {
-                    ast::Number::Integer { value } => ConstantData::Integer {
-                        value: value.clone(),
-                    },
-                    ast::Number::Float { value } => ConstantData::Float { value: *value },
-                    ast::Number::Complex { real, imag } => ConstantData::Complex {
-                        value: Complex64::new(*real, *imag),
-                    },
-                };
-                self.emit_constant(const_value);
+            Constant { value, .. } => {
+                self.emit_constant(compile_constant(value));
             }
-            List { elements } => {
-                let size = elements.len() as u32;
-                let must_unpack = self.gather_elements(elements)?;
+            List { elts, .. } => {
+                let size = elts.len() as u32;
+                let must_unpack = self.gather_elements(elts)?;
                 self.emit(Instruction::BuildList {
                     size,
                     unpack: must_unpack,
                 });
             }
-            Tuple { elements } => {
-                let size = elements.len() as u32;
-                let must_unpack = self.gather_elements(elements)?;
+            Tuple { elts, .. } => {
+                let size = elts.len() as u32;
+                let must_unpack = self.gather_elements(elts)?;
                 self.emit(Instruction::BuildTuple {
                     size,
                     unpack: must_unpack,
                 });
             }
-            Set { elements } => {
-                let size = elements.len() as u32;
-                let must_unpack = self.gather_elements(elements)?;
+            Set { elts, .. } => {
+                let size = elts.len() as u32;
+                let must_unpack = self.gather_elements(elts)?;
                 self.emit(Instruction::BuildSet {
                     size,
                     unpack: must_unpack,
                 });
             }
-            Dict { elements } => {
-                self.compile_dict(elements)?;
+            Dict { keys, values } => {
+                self.compile_dict(keys, values)?;
             }
-            Slice { elements } => {
-                let step = elements.len() >= 3;
-                for element in elements {
-                    self.compile_expression(element)?;
+            Slice { lower, upper, step } => {
+                let mut compile_bound = |bound: Option<&ast::Expr>| match bound {
+                    Some(exp) => self.compile_expression(exp),
+                    None => {
+                        self.emit_constant(ConstantData::None);
+                        Ok(())
+                    }
+                };
+                compile_bound(lower.as_deref())?;
+                compile_bound(upper.as_deref())?;
+                if let Some(step) = step {
+                    self.compile_expression(step)?;
                 }
+                let step = step.is_some();
                 self.emit(Instruction::BuildSlice { step });
             }
             Yield { value } => {
@@ -1926,28 +1920,36 @@ impl Compiler {
                 self.emit_constant(ConstantData::None);
                 self.emit(Instruction::YieldFrom);
             }
-            True => {
-                self.emit_constant(ConstantData::Boolean { value: true });
-            }
-            False => {
-                self.emit_constant(ConstantData::Boolean { value: false });
-            }
-            ast::ExpressionType::None => {
-                self.emit_constant(ConstantData::None);
-            }
-            Ellipsis => {
-                self.emit_constant(ConstantData::Ellipsis);
-            }
-            ast::ExpressionType::String { value } => {
-                self.compile_string(value)?;
+            ast::ExprKind::JoinedStr { values } => {
+                if let Some(value) = try_get_constant_string(values) {
+                    self.emit_constant(ConstantData::Str { value })
+                } else {
+                    for value in values {
+                        self.compile_expression(value)?;
+                    }
+                    self.emit(Instruction::BuildString {
+                        size: values.len() as u32,
+                    })
+                }
             }
-            Bytes { value } => {
-                self.emit_constant(ConstantData::Bytes {
-                    value: value.clone(),
+            ast::ExprKind::FormattedValue {
+                value,
+                conversion,
+                format_spec,
+            } => {
+                match format_spec {
+                    Some(spec) => self.compile_expression(spec)?,
+                    None => self.emit_constant(ConstantData::Str {
+                        value: String::new(),
+                    }),
+                };
+                self.compile_expression(value)?;
+                self.emit(Instruction::FormatValue {
+                    conversion: compile_conversion_flag(*conversion),
                 });
             }
-            Identifier { name } => {
-                self.load_name(name);
+            Name { id, .. } => {
+                self.load_name(id);
             }
             Lambda { args, body } => {
                 let prev_ctx = self.ctx;
@@ -1974,13 +1976,83 @@ impl Compiler {
 
                 self.ctx = prev_ctx;
             }
-            Comprehension { kind, generators } => {
-                self.compile_comprehension(kind, generators)?;
+            ListComp { elt, generators } => {
+                self.compile_comprehension(
+                    "<listcomp>",
+                    Some(Instruction::BuildList {
+                        size: 0,
+                        unpack: false,
+                    }),
+                    generators,
+                    &mut |compiler| {
+                        compiler.compile_comprehension_element(elt)?;
+                        compiler.emit(Instruction::ListAppend {
+                            i: (1 + generators.len()) as u32,
+                        });
+                        Ok(())
+                    },
+                )?;
+            }
+            SetComp { elt, generators } => {
+                self.compile_comprehension(
+                    "<setcomp>",
+                    Some(Instruction::BuildSet {
+                        size: 0,
+                        unpack: false,
+                    }),
+                    generators,
+                    &mut |compiler| {
+                        compiler.compile_comprehension_element(elt)?;
+                        compiler.emit(Instruction::SetAdd {
+                            i: (1 + generators.len()) as u32,
+                        });
+                        Ok(())
+                    },
+                )?;
+            }
+            DictComp {
+                key,
+                value,
+                generators,
+            } => {
+                self.compile_comprehension(
+                    "<dictcomp>",
+                    Some(Instruction::BuildMap {
+                        size: 0,
+                        for_call: false,
+                        unpack: false,
+                    }),
+                    generators,
+                    &mut |compiler| {
+                        // changed evaluation order for Py38 named expression PEP 572
+                        compiler.compile_expression(key)?;
+                        compiler.compile_expression(value)?;
+
+                        compiler.emit(Instruction::MapAddRev {
+                            i: (1 + generators.len()) as u32,
+                        });
+
+                        Ok(())
+                    },
+                )?;
+            }
+            GeneratorExp { elt, generators } => {
+                self.compile_comprehension("<genexpr>", None, generators, &mut |compiler| {
+                    compiler.compile_comprehension_element(elt)?;
+                    compiler.mark_generator();
+                    compiler.emit(Instruction::YieldValue);
+                    compiler.emit(Instruction::Pop);
+
+                    Ok(())
+                })?;
             }
+            // Comprehension { kind, generators } => {
+            //     self.compile_comprehension(kind, generators)?;
+            // }
             Starred { .. } => {
                 return Err(self.error(CompileErrorType::InvalidStarExpr));
             }
-            IfExpression { test, body, orelse } => {
+            IfExp { test, body, orelse } => {
                 let else_block = self.new_block();
                 let after_block = self.new_block();
                 self.compile_jump_if(test, false, else_block)?;
@@ -1999,10 +2071,10 @@ impl Compiler {
                 self.switch_to_block(after_block);
             }
 
-            NamedExpression { left, right } => {
-                self.compile_expression(right)?;
+            NamedExpr { target, value } => {
+                self.compile_expression(value)?;
                 self.emit(Instruction::Duplicate);
-                self.compile_store(left)?;
+                self.compile_store(target)?;
             }
         }
         Ok(())
@@ -2010,20 +2082,21 @@ impl Compiler {
 
     fn compile_keywords(&mut self, keywords: &[ast::Keyword]) -> CompileResult<()> {
         let mut size = 0;
-        for (is_unpacking, subkeywords) in &keywords.iter().group_by(|e| e.name.is_none()) {
+        let groupby = keywords.iter().group_by(|e| e.node.arg.is_none());
+        for (is_unpacking, subkeywords) in &groupby {
             if is_unpacking {
                 for keyword in subkeywords {
-                    self.compile_expression(&keyword.value)?;
+                    self.compile_expression(&keyword.node.value)?;
                     size += 1;
                 }
             } else {
                 let mut subsize = 0;
                 for keyword in subkeywords {
-                    if let Some(name) = &keyword.name {
+                    if let Some(name) = &keyword.node.arg {
                         self.emit_constant(ConstantData::Str {
                             value: name.to_owned(),
                         });
-                        self.compile_expression(&keyword.value)?;
+                        self.compile_expression(&keyword.node.value)?;
                         subsize += 1;
                     }
                 }
@@ -2047,8 +2120,8 @@ impl Compiler {
 
     fn compile_call(
         &mut self,
-        function: &ast::Expression,
-        args: &[ast::Expression],
+        function: &ast::Expr,
+        args: &[ast::Expr],
         keywords: &[ast::Keyword],
     ) -> CompileResult<()> {
         self.compile_expression(function)?;
@@ -2056,7 +2129,7 @@ impl Compiler {
 
         // Normal arguments:
         let must_unpack = self.gather_elements(args)?;
-        let has_double_star = keywords.iter().any(|k| k.name.is_none());
+        let has_double_star = keywords.iter().any(|k| k.node.arg.is_none());
 
         if must_unpack || has_double_star {
             // Create a tuple with positional args:
@@ -2077,7 +2150,7 @@ impl Compiler {
             if !keywords.is_empty() {
                 let mut kwarg_names = vec![];
                 for keyword in keywords {
-                    if let Some(name) = &keyword.name {
+                    if let Some(name) = &keyword.node.arg {
                         kwarg_names.push(ConstantData::Str {
                             value: name.to_owned(),
                         });
@@ -2085,7 +2158,7 @@ impl Compiler {
                         // This means **kwargs!
                         panic!("name must be set");
                     }
-                    self.compile_expression(&keyword.value)?;
+                    self.compile_expression(&keyword.node.value)?;
                 }
 
                 self.emit_constant(ConstantData::Tuple {
@@ -2101,14 +2174,14 @@ impl Compiler {
 
     // Given a vector of expr / star expr generate code which gives either
     // a list of expressions on the stack, or a list of tuples.
-    fn gather_elements(&mut self, elements: &[ast::Expression]) -> CompileResult<bool> {
+    fn gather_elements(&mut self, elements: &[ast::Expr]) -> CompileResult<bool> {
         // First determine if we have starred elements:
         let has_stars = elements
             .iter()
-            .any(|e| matches!(e.node, ast::ExpressionType::Starred { .. }));
+            .any(|e| matches!(e.node, ast::ExprKind::Starred { .. }));
 
         for element in elements {
-            if let ast::ExpressionType::Starred { value } = &element.node {
+            if let ast::ExprKind::Starred { value, .. } = &element.node {
                 self.compile_expression(value)?;
             } else {
                 self.compile_expression(element)?;
@@ -2124,10 +2197,24 @@ impl Compiler {
         Ok(has_stars)
     }
 
+    fn compile_comprehension_element(&mut self, element: &ast::Expr) -> CompileResult<()> {
+        self.compile_expression(element).map_err(|e| {
+            if let CompileErrorType::InvalidStarExpr = e.error {
+                self.error(CompileErrorType::SyntaxError(
+                    "iterable unpacking cannot be used in comprehension".to_owned(),
+                ))
+            } else {
+                e
+            }
+        })
+    }
+
     fn compile_comprehension(
         &mut self,
-        kind: &ast::ComprehensionKind,
+        name: &str,
+        init_collection: Option<Instruction>,
         generators: &[ast::Comprehension],
+        compile_element: &dyn Fn(&mut Self) -> CompileResult<()>,
     ) -> CompileResult<()> {
         let prev_ctx = self.ctx;
 
@@ -2140,50 +2227,20 @@ impl Compiler {
         // We must have at least one generator:
         assert!(!generators.is_empty());
 
-        let name = match kind {
-            ast::ComprehensionKind::GeneratorExpression { .. } => "<genexpr>",
-            ast::ComprehensionKind::List { .. } => "<listcomp>",
-            ast::ComprehensionKind::Set { .. } => "<setcomp>",
-            ast::ComprehensionKind::Dict { .. } => "<dictcomp>",
-        }
-        .to_owned();
-
         // Create magnificent function <listcomp>:
         self.push_output(
             bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED,
             1,
             1,
             0,
-            name.clone(),
+            name.to_owned(),
         );
         let arg0 = self.varname(".0");
 
-        // if this is a generator expression, we need to insert a LoadConst(None) before we return;
-        // the other kinds load their collection types below
-        let is_genexpr = matches!(kind, ast::ComprehensionKind::GeneratorExpression { .. });
-
+        let return_none = init_collection.is_none();
         // Create empty object of proper type:
-        match kind {
-            ast::ComprehensionKind::GeneratorExpression { .. } => {}
-            ast::ComprehensionKind::List { .. } => {
-                self.emit(Instruction::BuildList {
-                    size: 0,
-                    unpack: false,
-                });
-            }
-            ast::ComprehensionKind::Set { .. } => {
-                self.emit(Instruction::BuildSet {
-                    size: 0,
-                    unpack: false,
-                });
-            }
-            ast::ComprehensionKind::Dict { .. } => {
-                self.emit(Instruction::BuildMap {
-                    size: 0,
-                    unpack: false,
-                    for_call: false,
-                });
-            }
+        if let Some(init_collection) = init_collection {
+            self.emit(init_collection)
         }
 
         let mut loop_labels = vec![];
@@ -2223,47 +2280,7 @@ impl Compiler {
             }
         }
 
-        let mut compile_element = |element| {
-            self.compile_expression(element).map_err(|e| {
-                if let CompileErrorType::InvalidStarExpr = e.error {
-                    self.error(CompileErrorType::SyntaxError(
-                        "iterable unpacking cannot be used in comprehension".to_owned(),
-                    ))
-                } else {
-                    e
-                }
-            })
-        };
-
-        match kind {
-            ast::ComprehensionKind::GeneratorExpression { element } => {
-                compile_element(element)?;
-                self.mark_generator();
-                self.emit(Instruction::YieldValue);
-                self.emit(Instruction::Pop);
-            }
-            ast::ComprehensionKind::List { element } => {
-                compile_element(element)?;
-                self.emit(Instruction::ListAppend {
-                    i: (1 + generators.len()) as u32,
-                });
-            }
-            ast::ComprehensionKind::Set { element } => {
-                compile_element(element)?;
-                self.emit(Instruction::SetAdd {
-                    i: (1 + generators.len()) as u32,
-                });
-            }
-            ast::ComprehensionKind::Dict { key, value } => {
-                // changed evaluation order for Py38 named expression PEP 572
-                self.compile_expression(key)?;
-                self.compile_expression(value)?;
-
-                self.emit(Instruction::MapAddRev {
-                    i: (1 + generators.len()) as u32,
-                });
-            }
-        }
+        compile_element(self)?;
 
         for (loop_block, after_block) in loop_labels.iter().rev().copied() {
             // Repeat:
@@ -2274,7 +2291,7 @@ impl Compiler {
             self.emit(Instruction::PopBlock);
         }
 
-        if is_genexpr {
+        if return_none {
             self.emit_constant(ConstantData::None)
         }
 
@@ -2297,7 +2314,9 @@ impl Compiler {
         });
 
         // List comprehension function name:
-        self.emit_constant(ConstantData::Str { value: name });
+        self.emit_constant(ConstantData::Str {
+            value: name.to_owned(),
+        });
 
         // Turn code object into function object:
         self.emit(Instruction::MakeFunction(funcflags));
@@ -2313,54 +2332,12 @@ impl Compiler {
         Ok(())
     }
 
-    fn compile_string(&mut self, string: &ast::StringGroup) -> CompileResult<()> {
-        if let Some(value) = try_get_constant_string(string) {
-            self.emit_constant(ConstantData::Str { value });
-        } else {
-            match string {
-                ast::StringGroup::Joined { values } => {
-                    for value in values {
-                        self.compile_string(value)?;
-                    }
-                    self.emit(Instruction::BuildString {
-                        size: values.len() as u32,
-                    })
-                }
-                ast::StringGroup::Constant { value } => {
-                    self.emit_constant(ConstantData::Str {
-                        value: value.to_owned(),
-                    });
-                }
-                ast::StringGroup::FormattedValue {
-                    value,
-                    conversion,
-                    spec,
-                } => {
-                    match spec {
-                        Some(spec) => self.compile_string(spec)?,
-                        None => self.emit_constant(ConstantData::Str {
-                            value: String::new(),
-                        }),
-                    };
-                    self.compile_expression(value)?;
-                    self.emit(Instruction::FormatValue {
-                        conversion: compile_conversion_flag(*conversion),
-                    });
-                }
-            }
-        }
-        Ok(())
-    }
-
-    fn compile_future_features(
-        &mut self,
-        features: &[ast::ImportSymbol],
-    ) -> Result<(), CompileError> {
+    fn compile_future_features(&mut self, features: &[ast::Alias]) -> Result<(), CompileError> {
         if self.done_with_future_stmts {
             return Err(self.error(CompileErrorType::InvalidFuturePlacement));
         }
         for feature in features {
-            match &*feature.symbol {
+            match &*feature.name {
                 // Python 3 features; we've already implemented them by default
                 "nested_scopes" | "generators" | "division" | "absolute_import"
                 | "with_statement" | "print_function" | "unicode_literals" => {}
@@ -2447,34 +2424,38 @@ impl Compiler {
     }
 }
 
-fn get_doc(body: &[ast::Statement]) -> (&[ast::Statement], Option<String>) {
+fn get_doc(body: &[ast::Stmt]) -> (&[ast::Stmt], Option<String>) {
     if let Some((val, body_rest)) = body.split_first() {
-        if let ast::StatementType::Expression { ref expression } = val.node {
-            if let ast::ExpressionType::String { value } = &expression.node {
-                if let Some(value) = try_get_constant_string(value) {
-                    return (body_rest, Some(value));
-                }
+        if let ast::StmtKind::Expr { value } = &val.node {
+            if let Some(doc) = try_get_constant_string(std::slice::from_ref(value)) {
+                return (body_rest, Some(doc));
             }
         }
     }
     (body, None)
 }
 
-fn try_get_constant_string(string: &ast::StringGroup) -> Option<String> {
-    fn get_constant_string_inner(out_string: &mut String, string: &ast::StringGroup) -> bool {
-        match string {
-            ast::StringGroup::Constant { value } => {
-                out_string.push_str(&value);
+fn try_get_constant_string(values: &[ast::Expr]) -> Option<String> {
+    fn get_constant_string_inner(out_string: &mut String, value: &ast::Expr) -> bool {
+        match &value.node {
+            ast::ExprKind::Constant {
+                value: ast::Constant::Str(s),
+                ..
+            } => {
+                out_string.push_str(s);
                 true
             }
-            ast::StringGroup::Joined { values } => values
+            ast::ExprKind::JoinedStr { values } => values
                 .iter()
                 .all(|value| get_constant_string_inner(out_string, value)),
-            ast::StringGroup::FormattedValue { .. } => false,
+            _ => false,
         }
     }
     let mut out_string = String::new();
-    if get_constant_string_inner(&mut out_string, string) {
+    if values
+        .iter()
+        .all(|v| get_constant_string_inner(&mut out_string, v))
+    {
         Some(out_string)
     } else {
         None
@@ -2496,6 +2477,24 @@ fn compile_conversion_flag(
     }
 }
 
+fn compile_constant(value: &ast::Constant) -> ConstantData {
+    match value {
+        ast::Constant::None => ConstantData::None,
+        ast::Constant::Bool(b) => ConstantData::Boolean { value: *b },
+        ast::Constant::Str(s) => ConstantData::Str { value: s.clone() },
+        ast::Constant::Bytes(b) => ConstantData::Bytes { value: b.clone() },
+        ast::Constant::Int(i) => ConstantData::Integer { value: i.clone() },
+        ast::Constant::Tuple(t) => ConstantData::Tuple {
+            elements: t.iter().map(compile_constant).collect(),
+        },
+        ast::Constant::Float(f) => ConstantData::Float { value: *f },
+        ast::Constant::Complex { real, imag } => ConstantData::Complex {
+            value: Complex64::new(*real, *imag),
+        },
+        ast::Constant::Ellipsis => ConstantData::Ellipsis,
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::{CompileOpts, Compiler};
diff --git a/src/error.rs b/src/error.rs
index f1fa58e5..6d4b3d98 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -17,8 +17,6 @@ pub enum CompileErrorType {
     Assign(&'static str),
     /// Invalid delete
     Delete(&'static str),
-    /// Expected an expression got a statement
-    ExpectExpr,
     SyntaxError(String),
     /// Multiple `*` detected
     MultipleStarArgs,
@@ -45,7 +43,6 @@ impl fmt::Display for CompileErrorType {
         match self {
             CompileErrorType::Assign(target) => write!(f, "can't assign to {}", target),
             CompileErrorType::Delete(target) => write!(f, "can't delete {}", target),
-            CompileErrorType::ExpectExpr => write!(f, "Expecting expression, got statement"),
             CompileErrorType::SyntaxError(err) => write!(f, "{}", err.as_str()),
             CompileErrorType::MultipleStarArgs => {
                 write!(f, "two starred expressions in assignment")
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 4f22b8df..096e2dd3 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -12,19 +12,15 @@ use crate::IndexMap;
 use rustpython_ast::{self as ast, Location};
 use std::fmt;
 
-pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolTable, SymbolTableError> {
-    let mut builder = SymbolTableBuilder::default();
-    builder.prepare();
-    builder.scan_program(program)?;
+pub fn make_symbol_table(program: &[ast::Stmt]) -> SymbolTableResult<SymbolTable> {
+    let mut builder = SymbolTableBuilder::new();
+    builder.scan_statements(program)?;
     builder.finish()
 }
 
-pub fn statements_to_symbol_table(
-    statements: &[ast::Statement],
-) -> Result<SymbolTable, SymbolTableError> {
-    let mut builder = SymbolTableBuilder::default();
-    builder.prepare();
-    builder.scan_statements(statements)?;
+pub fn make_symbol_table_expr(expr: &ast::Expr) -> SymbolTableResult<SymbolTable> {
+    let mut builder = SymbolTableBuilder::new();
+    builder.scan_expression(expr, ExpressionContext::Load)?;
     builder.finish()
 }
 
@@ -179,7 +175,7 @@ impl SymbolTableError {
     }
 }
 
-type SymbolTableResult = Result<(), SymbolTableError>;
+type SymbolTableResult<T = ()> = Result<T, SymbolTableError>;
 
 impl SymbolTable {
     pub fn lookup(&self, name: &str) -> Option<&Symbol> {
@@ -540,7 +536,6 @@ enum SymbolUsage {
     Iter,
 }
 
-#[derive(Default)]
 struct SymbolTableBuilder {
     // Scope stack.
     tables: Vec<SymbolTable>,
@@ -560,11 +555,13 @@ enum ExpressionContext {
 }
 
 impl SymbolTableBuilder {
-    fn prepare(&mut self) {
-        self.enter_scope("top", SymbolTableType::Module, 0)
+    fn new() -> Self {
+        let mut this = Self { tables: vec![] };
+        this.enter_scope("top", SymbolTableType::Module, 0);
+        this
     }
 
-    fn finish(&mut self) -> Result<SymbolTable, SymbolTableError> {
+    fn finish(mut self) -> Result<SymbolTable, SymbolTableError> {
         assert_eq!(self.tables.len(), 1);
         let mut symbol_table = self.tables.pop().unwrap();
         analyze_symbol_table(&mut symbol_table)?;
@@ -587,50 +584,45 @@ impl SymbolTableBuilder {
         self.tables.last_mut().unwrap().sub_tables.push(table);
     }
 
-    fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult {
-        self.scan_statements(&program.statements)?;
-        Ok(())
-    }
-
-    fn scan_statements(&mut self, statements: &[ast::Statement]) -> SymbolTableResult {
+    fn scan_statements(&mut self, statements: &[ast::Stmt]) -> SymbolTableResult {
         for statement in statements {
             self.scan_statement(statement)?;
         }
         Ok(())
     }
 
-    fn scan_parameters(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
+    fn scan_parameters(&mut self, parameters: &[ast::Arg]) -> SymbolTableResult {
         for parameter in parameters {
             self.scan_parameter(parameter)?;
         }
         Ok(())
     }
 
-    fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
-        let usage = if parameter.annotation.is_some() {
+    fn scan_parameter(&mut self, parameter: &ast::Arg) -> SymbolTableResult {
+        let usage = if parameter.node.annotation.is_some() {
             SymbolUsage::AnnotationParameter
         } else {
             SymbolUsage::Parameter
         };
-        self.register_name(&parameter.arg, usage, parameter.location)
+        self.register_name(&parameter.node.arg, usage, parameter.location)
     }
 
-    fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
+    fn scan_parameters_annotations(&mut self, parameters: &[ast::Arg]) -> SymbolTableResult {
         for parameter in parameters {
             self.scan_parameter_annotation(parameter)?;
         }
         Ok(())
     }
 
-    fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
-        if let Some(annotation) = &parameter.annotation {
+    fn scan_parameter_annotation(&mut self, parameter: &ast::Arg) -> SymbolTableResult {
+        if let Some(annotation) = &parameter.node.annotation {
             self.scan_expression(&annotation, ExpressionContext::Load)?;
         }
         Ok(())
     }
 
-    fn scan_statement(&mut self, statement: &ast::Statement) -> SymbolTableResult {
-        use ast::StatementType::*;
+    fn scan_statement(&mut self, statement: &ast::Stmt) -> SymbolTableResult {
+        use ast::StmtKind::*;
         let location = statement.location;
         match &statement.node {
             Global { names } => {
@@ -650,6 +642,14 @@ impl SymbolTableBuilder {
                 decorator_list,
                 returns,
                 ..
+            }
+            | AsyncFunctionDef {
+                name,
+                body,
+                args,
+                decorator_list,
+                returns,
+                ..
             } => {
                 self.scan_expressions(decorator_list, ExpressionContext::Load)?;
                 self.register_name(name, SymbolUsage::Assigned, location)?;
@@ -676,20 +676,16 @@ impl SymbolTableBuilder {
                 self.leave_scope();
                 self.scan_expressions(bases, ExpressionContext::Load)?;
                 for keyword in keywords {
-                    self.scan_expression(&keyword.value, ExpressionContext::Load)?;
+                    self.scan_expression(&keyword.node.value, ExpressionContext::Load)?;
                 }
                 self.scan_expressions(decorator_list, ExpressionContext::Load)?;
                 self.register_name(name, SymbolUsage::Assigned, location)?;
             }
-            Expression { expression } => {
-                self.scan_expression(expression, ExpressionContext::Load)?
-            }
+            Expr { value } => self.scan_expression(value, ExpressionContext::Load)?,
             If { test, body, orelse } => {
                 self.scan_expression(test, ExpressionContext::Load)?;
                 self.scan_statements(body)?;
-                if let Some(code) = orelse {
-                    self.scan_statements(code)?;
-                }
+                self.scan_statements(orelse)?;
             }
             For {
                 target,
@@ -697,33 +693,36 @@ impl SymbolTableBuilder {
                 body,
                 orelse,
                 ..
+            }
+            | AsyncFor {
+                target,
+                iter,
+                body,
+                orelse,
+                ..
             } => {
                 self.scan_expression(target, ExpressionContext::Store)?;
                 self.scan_expression(iter, ExpressionContext::Load)?;
                 self.scan_statements(body)?;
-                if let Some(code) = orelse {
-                    self.scan_statements(code)?;
-                }
+                self.scan_statements(orelse)?;
             }
             While { test, body, orelse } => {
                 self.scan_expression(test, ExpressionContext::Load)?;
                 self.scan_statements(body)?;
-                if let Some(code) = orelse {
-                    self.scan_statements(code)?;
-                }
+                self.scan_statements(orelse)?;
             }
             Break | Continue | Pass => {
                 // No symbols here.
             }
             Import { names } | ImportFrom { names, .. } => {
                 for name in names {
-                    if let Some(alias) = &name.alias {
+                    if let Some(alias) = &name.asname {
                         // `import mymodule as myalias`
                         self.register_name(alias, SymbolUsage::Imported, location)?;
                     } else {
                         // `import module`
                         self.register_name(
-                            name.symbol.split('.').next().unwrap(),
+                            name.name.split('.').next().unwrap(),
                             SymbolUsage::Imported,
                             location,
                         )?;
@@ -744,7 +743,7 @@ impl SymbolTableBuilder {
             Delete { targets } => {
                 self.scan_expressions(targets, ExpressionContext::Delete)?;
             }
-            Assign { targets, value } => {
+            Assign { targets, value, .. } => {
                 self.scan_expressions(targets, ExpressionContext::Store)?;
                 self.scan_expression(value, ExpressionContext::Load)?;
             }
@@ -756,19 +755,23 @@ impl SymbolTableBuilder {
                 target,
                 annotation,
                 value,
+                simple,
             } => {
                 // https://github.com/python/cpython/blob/master/Python/symtable.c#L1233
-                if let ast::ExpressionType::Identifier { ref name } = target.node {
-                    self.register_name(name, SymbolUsage::AnnotationAssigned, location)?;
-                } else {
-                    self.scan_expression(target, ExpressionContext::Store)?;
+                match &target.node {
+                    ast::ExprKind::Name { id, .. } if *simple => {
+                        self.register_name(id, SymbolUsage::AnnotationAssigned, location)?;
+                    }
+                    _ => {
+                        self.scan_expression(target, ExpressionContext::Store)?;
+                    }
                 }
                 self.scan_expression(annotation, ExpressionContext::Load)?;
                 if let Some(value) = value {
                     self.scan_expression(value, ExpressionContext::Load)?;
                 }
             }
-            With { items, body, .. } => {
+            With { items, body, .. } | AsyncWith { items, body, .. } => {
                 for item in items {
                     self.scan_expression(&item.context_expr, ExpressionContext::Load)?;
                     if let Some(expression) = &item.optional_vars {
@@ -785,23 +788,20 @@ impl SymbolTableBuilder {
             } => {
                 self.scan_statements(body)?;
                 for handler in handlers {
-                    if let Some(expression) = &handler.typ {
+                    let ast::ExcepthandlerKind::ExceptHandler { type_, name, body } = &handler.node;
+                    if let Some(expression) = type_ {
                         self.scan_expression(expression, ExpressionContext::Load)?;
                     }
-                    if let Some(name) = &handler.name {
+                    if let Some(name) = name {
                         self.register_name(name, SymbolUsage::Assigned, location)?;
                     }
-                    self.scan_statements(&handler.body)?;
-                }
-                if let Some(code) = orelse {
-                    self.scan_statements(code)?;
-                }
-                if let Some(code) = finalbody {
-                    self.scan_statements(code)?;
+                    self.scan_statements(body)?;
                 }
+                self.scan_statements(orelse)?;
+                self.scan_statements(finalbody)?;
             }
-            Raise { exception, cause } => {
-                if let Some(expression) = exception {
+            Raise { exc, cause } => {
+                if let Some(expression) = exc {
                     self.scan_expression(expression, ExpressionContext::Load)?;
                 }
                 if let Some(expression) = cause {
@@ -814,7 +814,7 @@ impl SymbolTableBuilder {
 
     fn scan_expressions(
         &mut self,
-        expressions: &[ast::Expression],
+        expressions: &[ast::Expr],
         context: ExpressionContext,
     ) -> SymbolTableResult {
         for expression in expressions {
@@ -825,31 +825,34 @@ impl SymbolTableBuilder {
 
     fn scan_expression(
         &mut self,
-        expression: &ast::Expression,
+        expression: &ast::Expr,
         context: ExpressionContext,
     ) -> SymbolTableResult {
-        use ast::ExpressionType::*;
+        use ast::ExprKind::*;
         let location = expression.location;
         match &expression.node {
-            Binop { a, b, .. } => {
-                self.scan_expression(a, context)?;
-                self.scan_expression(b, context)?;
+            BinOp { left, right, .. } => {
+                self.scan_expression(left, context)?;
+                self.scan_expression(right, context)?;
             }
             BoolOp { values, .. } => {
                 self.scan_expressions(values, context)?;
             }
-            Compare { vals, .. } => {
-                self.scan_expressions(vals, context)?;
+            Compare {
+                left, comparators, ..
+            } => {
+                self.scan_expression(left, context)?;
+                self.scan_expressions(comparators, context)?;
             }
-            Subscript { a, b } => {
-                self.scan_expression(a, ExpressionContext::Load)?;
-                self.scan_expression(b, ExpressionContext::Load)?;
+            Subscript { value, slice, .. } => {
+                self.scan_expression(value, ExpressionContext::Load)?;
+                self.scan_expression(slice, ExpressionContext::Load)?;
             }
             Attribute { value, .. } => {
                 self.scan_expression(value, ExpressionContext::Load)?;
             }
-            Dict { elements } => {
-                for (key, value) in elements {
+            Dict { keys, values } => {
+                for (key, value) in keys.iter().zip(values) {
                     if let Some(key) = key {
                         self.scan_expression(key, context)?;
                     } else {
@@ -869,109 +872,95 @@ impl SymbolTableBuilder {
             YieldFrom { value } => {
                 self.scan_expression(value, context)?;
             }
-            Unop { a, .. } => {
-                self.scan_expression(a, context)?;
+            UnaryOp { operand, .. } => {
+                self.scan_expression(operand, context)?;
             }
-            True | False | None | Ellipsis => {}
-            Number { .. } => {}
-            Starred { value } => {
+            Constant { .. } => {}
+            Starred { value, .. } => {
                 self.scan_expression(value, context)?;
             }
-            Bytes { .. } => {}
-            Tuple { elements } | Set { elements } | List { elements } | Slice { elements } => {
-                self.scan_expressions(elements, context)?;
-            }
-            Comprehension { kind, generators } => {
-                // Comprehensions are compiled as functions, so create a scope for them:
-                let scope_name = match **kind {
-                    ast::ComprehensionKind::GeneratorExpression { .. } => "genexpr",
-                    ast::ComprehensionKind::List { .. } => "listcomp",
-                    ast::ComprehensionKind::Set { .. } => "setcomp",
-                    ast::ComprehensionKind::Dict { .. } => "dictcomp",
-                };
-
-                self.enter_scope(scope_name, SymbolTableType::Comprehension, location.row());
-
-                // Register the passed argument to the generator function as the name ".0"
-                self.register_name(".0", SymbolUsage::Parameter, location)?;
-
-                match **kind {
-                    ast::ComprehensionKind::GeneratorExpression { ref element }
-                    | ast::ComprehensionKind::List { ref element }
-                    | ast::ComprehensionKind::Set { ref element } => {
-                        self.scan_expression(element, ExpressionContext::Load)?;
-                    }
-                    ast::ComprehensionKind::Dict { ref key, ref value } => {
-                        self.scan_expression(&key, ExpressionContext::Load)?;
-                        self.scan_expression(&value, ExpressionContext::Load)?;
-                    }
+            Tuple { elts, .. } | Set { elts, .. } | List { elts, .. } => {
+                self.scan_expressions(elts, context)?;
+            }
+            Slice { lower, upper, step } => {
+                if let Some(lower) = lower {
+                    self.scan_expression(lower, context)?;
                 }
-
-                let mut is_first_generator = true;
-                for generator in generators {
-                    self.scan_expression(&generator.target, ExpressionContext::Iter)?;
-                    if is_first_generator {
-                        is_first_generator = false;
-                    } else {
-                        self.scan_expression(
-                            &generator.iter,
-                            ExpressionContext::IterDefinitionExp,
-                        )?;
-                    }
-
-                    for if_expr in &generator.ifs {
-                        self.scan_expression(if_expr, ExpressionContext::Load)?;
-                    }
+                if let Some(upper) = upper {
+                    self.scan_expression(upper, context)?;
                 }
-
-                self.leave_scope();
-
-                // The first iterable is passed as an argument into the created function:
-                assert!(!generators.is_empty());
-                self.scan_expression(&generators[0].iter, ExpressionContext::IterDefinitionExp)?;
+                if let Some(step) = step {
+                    self.scan_expression(step, context)?;
+                }
+            }
+            GeneratorExp { elt, generators } => {
+                self.scan_comprehension("genexpr", elt, None, generators, location)?;
+            }
+            ListComp { elt, generators } => {
+                self.scan_comprehension("genexpr", elt, None, generators, location)?;
+            }
+            SetComp { elt, generators } => {
+                self.scan_comprehension("genexpr", elt, None, generators, location)?;
+            }
+            DictComp {
+                key,
+                value,
+                generators,
+            } => {
+                self.scan_comprehension("genexpr", key, Some(value), generators, location)?;
             }
             Call {
-                function,
+                func,
                 args,
                 keywords,
             } => {
                 match context {
                     ExpressionContext::IterDefinitionExp => {
-                        self.scan_expression(function, ExpressionContext::IterDefinitionExp)?;
+                        self.scan_expression(func, ExpressionContext::IterDefinitionExp)?;
                     }
                     _ => {
-                        self.scan_expression(function, ExpressionContext::Load)?;
+                        self.scan_expression(func, ExpressionContext::Load)?;
                     }
                 }
 
                 self.scan_expressions(args, ExpressionContext::Load)?;
                 for keyword in keywords {
-                    self.scan_expression(&keyword.value, ExpressionContext::Load)?;
+                    self.scan_expression(&keyword.node.value, ExpressionContext::Load)?;
                 }
             }
-            String { value } => {
-                self.scan_string_group(value)?;
+            FormattedValue {
+                value, format_spec, ..
+            } => {
+                self.scan_expression(value, ExpressionContext::Load)?;
+                if let Some(spec) = format_spec {
+                    self.scan_expression(spec, ExpressionContext::Load)?;
+                }
+            }
+            JoinedStr { values } => {
+                for value in values {
+                    self.scan_expression(value, ExpressionContext::Load)?;
+                }
             }
-            Identifier { name } => {
+            Name { id, .. } => {
                 // Determine the contextual usage of this symbol:
                 match context {
                     ExpressionContext::Delete => {
-                        self.register_name(name, SymbolUsage::Assigned, location)?;
-                        self.register_name(name, SymbolUsage::Used, location)?;
+                        self.register_name(id, SymbolUsage::Assigned, location)?;
+                        self.register_name(id, SymbolUsage::Used, location)?;
                     }
                     ExpressionContext::Load | ExpressionContext::IterDefinitionExp => {
-                        self.register_name(name, SymbolUsage::Used, location)?;
+                        self.register_name(id, SymbolUsage::Used, location)?;
                     }
                     ExpressionContext::Store => {
-                        self.register_name(name, SymbolUsage::Assigned, location)?;
+                        self.register_name(id, SymbolUsage::Assigned, location)?;
                     }
                     ExpressionContext::Iter => {
-                        self.register_name(name, SymbolUsage::Iter, location)?;
+                        self.register_name(id, SymbolUsage::Iter, location)?;
                     }
                 }
                 if context == ExpressionContext::Load
                     && self.tables.last().unwrap().typ == SymbolTableType::Function
-                    && name == "super"
+                    && id == "super"
                 {
                     self.register_name("__class__", SymbolUsage::Used, location)?;
                 }
@@ -988,13 +977,13 @@ impl SymbolTableBuilder {
                 }
                 self.leave_scope();
             }
-            IfExpression { test, body, orelse } => {
+            IfExp { test, body, orelse } => {
                 self.scan_expression(test, ExpressionContext::Load)?;
                 self.scan_expression(body, ExpressionContext::Load)?;
                 self.scan_expression(orelse, ExpressionContext::Load)?;
             }
 
-            NamedExpression { left, right } => {
+            NamedExpr { target, value } => {
                 // named expressions are not allowed in the definiton of
                 // comprehension iterator definitions
                 if let ExpressionContext::IterDefinitionExp = context {
@@ -1005,34 +994,77 @@ impl SymbolTableBuilder {
                     });
                 }
 
-                self.scan_expression(right, ExpressionContext::Load)?;
+                self.scan_expression(value, ExpressionContext::Load)?;
 
                 // special handling for assigned identifier in named expressions
                 // that are used in comprehensions. This required to correctly
                 // propagate the scope of the named assigned named and not to
                 // propagate inner names.
-                if let Identifier { name } = &left.node {
+                if let Name { id, .. } = &target.node {
                     let table = self.tables.last().unwrap();
                     if table.typ == SymbolTableType::Comprehension {
-                        self.register_name(name, SymbolUsage::AssignedNamedExprInCompr, location)?;
+                        self.register_name(id, SymbolUsage::AssignedNamedExprInCompr, location)?;
                     } else {
                         // omit one recursion. When the handling of an store changes for
                         // Identifiers this needs adapted - more forward safe would be
                         // calling scan_expression directly.
-                        self.register_name(name, SymbolUsage::Assigned, location)?;
+                        self.register_name(id, SymbolUsage::Assigned, location)?;
                     }
                 } else {
-                    self.scan_expression(left, ExpressionContext::Store)?;
+                    self.scan_expression(target, ExpressionContext::Store)?;
                 }
             }
         }
         Ok(())
     }
 
+    fn scan_comprehension(
+        &mut self,
+        scope_name: &str,
+        elt1: &ast::Expr,
+        elt2: Option<&ast::Expr>,
+        generators: &[ast::Comprehension],
+        location: Location,
+    ) -> SymbolTableResult {
+        // Comprehensions are compiled as functions, so create a scope for them:
+
+        self.enter_scope(scope_name, SymbolTableType::Comprehension, location.row());
+
+        // Register the passed argument to the generator function as the name ".0"
+        self.register_name(".0", SymbolUsage::Parameter, location)?;
+
+        self.scan_expression(elt1, ExpressionContext::Load)?;
+        if let Some(elt2) = elt2 {
+            self.scan_expression(elt2, ExpressionContext::Load)?;
+        }
+
+        let mut is_first_generator = true;
+        for generator in generators {
+            self.scan_expression(&generator.target, ExpressionContext::Iter)?;
+            if is_first_generator {
+                is_first_generator = false;
+            } else {
+                self.scan_expression(&generator.iter, ExpressionContext::IterDefinitionExp)?;
+            }
+
+            for if_expr in &generator.ifs {
+                self.scan_expression(if_expr, ExpressionContext::Load)?;
+            }
+        }
+
+        self.leave_scope();
+
+        // The first iterable is passed as an argument into the created function:
+        assert!(!generators.is_empty());
+        self.scan_expression(&generators[0].iter, ExpressionContext::IterDefinitionExp)?;
+
+        Ok(())
+    }
+
     fn enter_function(
         &mut self,
         name: &str,
-        args: &ast::Parameters,
+        args: &ast::Arguments,
         line_number: usize,
     ) -> SymbolTableResult {
         // Evaluate eventual default parameters:
@@ -1044,47 +1076,31 @@ impl SymbolTableBuilder {
         }
 
         // Annotations are scanned in outer scope:
+        self.scan_parameters_annotations(&args.posonlyargs)?;
         self.scan_parameters_annotations(&args.args)?;
         self.scan_parameters_annotations(&args.kwonlyargs)?;
-        if let ast::Varargs::Named(name) = &args.vararg {
+        if let Some(name) = &args.vararg {
             self.scan_parameter_annotation(name)?;
         }
-        if let ast::Varargs::Named(name) = &args.kwarg {
+        if let Some(name) = &args.kwarg {
             self.scan_parameter_annotation(name)?;
         }
 
         self.enter_scope(name, SymbolTableType::Function, line_number);
 
         // Fill scope with parameter names:
+        self.scan_parameters(&args.posonlyargs)?;
         self.scan_parameters(&args.args)?;
         self.scan_parameters(&args.kwonlyargs)?;
-        if let ast::Varargs::Named(name) = &args.vararg {
+        if let Some(name) = &args.vararg {
             self.scan_parameter(name)?;
         }
-        if let ast::Varargs::Named(name) = &args.kwarg {
+        if let Some(name) = &args.kwarg {
             self.scan_parameter(name)?;
         }
         Ok(())
     }
 
-    fn scan_string_group(&mut self, group: &ast::StringGroup) -> SymbolTableResult {
-        match group {
-            ast::StringGroup::Constant { .. } => {}
-            ast::StringGroup::FormattedValue { value, spec, .. } => {
-                self.scan_expression(value, ExpressionContext::Load)?;
-                if let Some(spec) = spec {
-                    self.scan_string_group(spec)?;
-                }
-            }
-            ast::StringGroup::Joined { values } => {
-                for subgroup in values {
-                    self.scan_string_group(subgroup)?;
-                }
-            }
-        }
-        Ok(())
-    }
-
     fn register_name(
         &mut self,
         name: &str,

From f1970bfbf97b6dd8656060ce0fa1baac99234dfa Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sun, 10 Jan 2021 20:21:10 -0600
Subject: [PATCH 188/459] Misc fixes

---
 src/compile.rs     | 18 ++++++++++--------
 src/symboltable.rs | 12 +++++-------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index bddf3d64..0278f1c2 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1082,7 +1082,12 @@ impl Compiler {
         }
         for var in &*code.freevars {
             let table = self.symbol_table_stack.last().unwrap();
-            let symbol = table.lookup(var).unwrap();
+            let symbol = table.lookup(var).unwrap_or_else(|| {
+                panic!(
+                    "couldn't look up var {} in {} in {}",
+                    var, code.obj_name, self.source_path
+                )
+            });
             let parent_code = self.code_stack.last().unwrap();
             let vars = match symbol.scope {
                 SymbolScope::Free => &parent_code.freevar_cache,
@@ -1984,7 +1989,7 @@ impl Compiler {
                         unpack: false,
                     }),
                     generators,
-                    &mut |compiler| {
+                    &|compiler| {
                         compiler.compile_comprehension_element(elt)?;
                         compiler.emit(Instruction::ListAppend {
                             i: (1 + generators.len()) as u32,
@@ -2001,7 +2006,7 @@ impl Compiler {
                         unpack: false,
                     }),
                     generators,
-                    &mut |compiler| {
+                    &|compiler| {
                         compiler.compile_comprehension_element(elt)?;
                         compiler.emit(Instruction::SetAdd {
                             i: (1 + generators.len()) as u32,
@@ -2023,7 +2028,7 @@ impl Compiler {
                         unpack: false,
                     }),
                     generators,
-                    &mut |compiler| {
+                    &|compiler| {
                         // changed evaluation order for Py38 named expression PEP 572
                         compiler.compile_expression(key)?;
                         compiler.compile_expression(value)?;
@@ -2037,7 +2042,7 @@ impl Compiler {
                 )?;
             }
             GeneratorExp { elt, generators } => {
-                self.compile_comprehension("<genexpr>", None, generators, &mut |compiler| {
+                self.compile_comprehension("<genexpr>", None, generators, &|compiler| {
                     compiler.compile_comprehension_element(elt)?;
                     compiler.mark_generator();
                     compiler.emit(Instruction::YieldValue);
@@ -2046,9 +2051,6 @@ impl Compiler {
                     Ok(())
                 })?;
             }
-            // Comprehension { kind, generators } => {
-            //     self.compile_comprehension(kind, generators)?;
-            // }
             Starred { .. } => {
                 return Err(self.error(CompileErrorType::InvalidStarExpr));
             }
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 096e2dd3..2245f336 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -358,15 +358,11 @@ impl SymbolTableAnalyzer {
     }
 
     fn found_in_outer_scope(&mut self, name: &str) -> Option<SymbolScope> {
-        // Interesting stuff about the __class__ variable:
-        // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
-        if name == "__class__" {
-            return Some(SymbolScope::Free);
-        }
-
         let mut decl_depth = None;
         for (i, (symbols, typ)) in self.tables.iter().rev().enumerate() {
-            if let SymbolTableType::Class | SymbolTableType::Module = typ {
+            if matches!(typ, SymbolTableType::Module)
+                || matches!(typ, SymbolTableType::Class if name != "__class__")
+            {
                 continue;
             }
             if let Some(sym) = symbols.get(name) {
@@ -958,6 +954,8 @@ impl SymbolTableBuilder {
                         self.register_name(id, SymbolUsage::Iter, location)?;
                     }
                 }
+                // Interesting stuff about the __class__ variable:
+                // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object
                 if context == ExpressionContext::Load
                     && self.tables.last().unwrap().typ == SymbolTableType::Function
                     && id == "super"

From 83484172749e114f85a1c98cb1a5a455385c5be0 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Tue, 26 Jan 2021 09:59:16 -0600
Subject: [PATCH 189/459] Implement name mangling

---
 src/compile.rs     | 37 ++++++++++++++++++++++++----------
 src/symboltable.rs | 50 +++++++++++++++++++++++++++++-----------------
 2 files changed, 59 insertions(+), 28 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 0278f1c2..4775dc18 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -5,16 +5,20 @@
 //!   https://github.com/python/cpython/blob/master/Python/compile.c
 //!   https://github.com/micropython/micropython/blob/master/py/compile.c
 
-use crate::error::{CompileError, CompileErrorType};
 use crate::ir::{self, CodeInfo};
 pub use crate::mode::Mode;
 use crate::symboltable::{make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable};
 use crate::IndexSet;
+use crate::{
+    error::{CompileError, CompileErrorType},
+    symboltable,
+};
 use itertools::Itertools;
 use num_complex::Complex64;
 use num_traits::ToPrimitive;
 use rustpython_ast as ast;
 use rustpython_bytecode::{self as bytecode, CodeObject, ConstantData, Instruction};
+use std::borrow::Cow;
 
 type CompileResult<T> = Result<T, CompileError>;
 
@@ -33,6 +37,7 @@ struct Compiler {
     current_qualified_path: Option<String>,
     done_with_future_stmts: bool,
     ctx: CompileContext,
+    class_name: Option<String>,
     opts: CompileOpts,
 }
 
@@ -170,6 +175,7 @@ impl Compiler {
                 in_class: false,
                 func: FunctionContext::NoFunction,
             },
+            class_name: None,
             opts,
         }
     }
@@ -260,10 +266,11 @@ impl Compiler {
         name: &str,
         cache: impl FnOnce(&mut CodeInfo) -> &mut IndexSet<String>,
     ) -> bytecode::NameIdx {
+        let name = self.mangle(name);
         let cache = cache(self.current_codeinfo());
         cache
-            .get_index_of(name)
-            .unwrap_or_else(|| cache.insert_full(name.to_owned()).0) as u32
+            .get_index_of(name.as_ref())
+            .unwrap_or_else(|| cache.insert_full(name.into_owned()).0) as u32
     }
 
     fn compile_program(
@@ -358,9 +365,14 @@ impl Compiler {
         self.compile_name(name, NameUsage::Store)
     }
 
+    fn mangle<'a>(&self, name: &'a str) -> Cow<'a, str> {
+        symboltable::mangle_name(self.class_name.as_deref(), name)
+    }
+
     fn compile_name(&mut self, name: &str, usage: NameUsage) {
+        let name = self.mangle(name);
         let symbol_table = self.symbol_table_stack.last().unwrap();
-        let symbol = symbol_table.lookup(name).expect(
+        let symbol = symbol_table.lookup(name.as_ref()).expect(
             "The symbol must be present in the symbol table, even when it is undefined in python.",
         );
         let info = self.code_stack.last_mut().unwrap();
@@ -394,8 +406,8 @@ impl Compiler {
             // SymbolScope::Unknown => NameOpType::Global,
         };
         let mut idx = cache
-            .get_index_of(name)
-            .unwrap_or_else(|| cache.insert_full(name.to_owned()).0);
+            .get_index_of(name.as_ref())
+            .unwrap_or_else(|| cache.insert_full(name.into_owned()).0);
         if let SymbolScope::Free = symbol.scope {
             idx += info.cellvar_cache.len();
         }
@@ -1033,7 +1045,7 @@ impl Compiler {
         for arg in args_iter {
             if let Some(annotation) = &arg.node.annotation {
                 self.emit_constant(ConstantData::Str {
-                    value: arg.node.arg.to_owned(),
+                    value: self.mangle(&arg.node.arg).into_owned(),
                 });
                 self.compile_expression(&annotation)?;
                 num_annotations += 1;
@@ -1153,9 +1165,13 @@ impl Compiler {
             loop_data: None,
         };
 
+        let prev_class_name = std::mem::replace(&mut self.class_name, Some(name.to_owned()));
+
         let qualified_name = self.create_qualified_name(name, "");
-        let old_qualified_path = self.current_qualified_path.take();
-        self.current_qualified_path = Some(qualified_name.clone());
+        let old_qualified_path = std::mem::replace(
+            &mut self.current_qualified_path,
+            Some(qualified_name.clone()),
+        );
 
         self.emit(Instruction::LoadBuildClass);
         self.push_output(bytecode::CodeFlags::empty(), 0, 0, 0, name.to_owned());
@@ -1201,6 +1217,7 @@ impl Compiler {
 
         let code = self.pop_code_object();
 
+        self.class_name = prev_class_name;
         self.current_qualified_path = old_qualified_path;
         self.ctx = prev_ctx;
 
@@ -1531,7 +1548,7 @@ impl Compiler {
             let annotations = self.name("__annotations__");
             self.emit(Instruction::LoadNameAny(annotations));
             self.emit_constant(ConstantData::Str {
-                value: id.to_owned(),
+                value: self.mangle(id).into_owned(),
             });
             self.emit(Instruction::StoreSubscript);
         } else {
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 2245f336..44c6df90 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -10,6 +10,7 @@ Inspirational file: https://github.com/python/cpython/blob/master/Python/symtabl
 use crate::error::{CompileError, CompileErrorType};
 use crate::IndexMap;
 use rustpython_ast::{self as ast, Location};
+use std::borrow::Cow;
 use std::fmt;
 
 pub fn make_symbol_table(program: &[ast::Stmt]) -> SymbolTableResult<SymbolTable> {
@@ -533,6 +534,7 @@ enum SymbolUsage {
 }
 
 struct SymbolTableBuilder {
+    class_name: Option<String>,
     // Scope stack.
     tables: Vec<SymbolTable>,
 }
@@ -552,7 +554,10 @@ enum ExpressionContext {
 
 impl SymbolTableBuilder {
     fn new() -> Self {
-        let mut this = Self { tables: vec![] };
+        let mut this = Self {
+            class_name: None,
+            tables: vec![],
+        };
         this.enter_scope("top", SymbolTableType::Module, 0);
         this
     }
@@ -664,12 +669,14 @@ impl SymbolTableBuilder {
                 decorator_list,
             } => {
                 self.enter_scope(name, SymbolTableType::Class, location.row());
+                let prev_class = std::mem::replace(&mut self.class_name, Some(name.to_owned()));
                 self.register_name("__module__", SymbolUsage::Assigned, location)?;
                 self.register_name("__qualname__", SymbolUsage::Assigned, location)?;
                 self.register_name("__doc__", SymbolUsage::Assigned, location)?;
                 self.register_name("__class__", SymbolUsage::Assigned, location)?;
                 self.scan_statements(body)?;
                 self.leave_scope();
+                self.class_name = prev_class;
                 self.scan_expressions(bases, ExpressionContext::Load)?;
                 for keyword in keywords {
                     self.scan_expression(&keyword.node.value, ExpressionContext::Load)?;
@@ -1108,15 +1115,14 @@ impl SymbolTableBuilder {
         let scope_depth = self.tables.len();
         let table = self.tables.last_mut().unwrap();
 
+        let name = mangle_name(self.class_name.as_deref(), name);
+
         // Some checks for the symbol that present on this scope level:
-        let symbol = if let Some(symbol) = table.symbols.get_mut(name) {
+        let symbol = if let Some(symbol) = table.symbols.get_mut(name.as_ref()) {
             // Role already set..
             match role {
                 SymbolUsage::Global => {
-                    if symbol.is_global() {
-                        // Ok
-                        symbol.scope = SymbolScope::GlobalExplicit;
-                    } else {
+                    if !symbol.is_global() {
                         return Err(SymbolTableError {
                             error: format!("name '{}' is used prior to global declaration", name),
                             location,
@@ -1172,8 +1178,8 @@ impl SymbolTableBuilder {
                 }
             }
             // Insert symbol when required:
-            let symbol = Symbol::new(name);
-            table.symbols.entry(name.to_owned()).or_insert(symbol)
+            let symbol = Symbol::new(name.as_ref());
+            table.symbols.entry(name.into_owned()).or_insert(symbol)
         };
 
         // Set proper flags on symbol:
@@ -1205,16 +1211,7 @@ impl SymbolTableBuilder {
                 symbol.is_assign_namedexpr_in_comprehension = true;
             }
             SymbolUsage::Global => {
-                if let SymbolScope::Unknown = symbol.scope {
-                    symbol.scope = SymbolScope::GlobalExplicit;
-                } else if symbol.is_global() {
-                    symbol.scope = SymbolScope::GlobalExplicit;
-                } else {
-                    return Err(SymbolTableError {
-                        error: format!("Symbol {} scope cannot be set to global, since its scope was already determined otherwise.", name),
-                        location,
-                    });
-                }
+                symbol.scope = SymbolScope::GlobalExplicit;
             }
             SymbolUsage::Used => {
                 symbol.is_referenced = true;
@@ -1239,3 +1236,20 @@ impl SymbolTableBuilder {
         Ok(())
     }
 }
+
+pub(crate) fn mangle_name<'a>(class_name: Option<&str>, name: &'a str) -> Cow<'a, str> {
+    let class_name = match class_name {
+        Some(n) => n,
+        None => return name.into(),
+    };
+    if !name.starts_with("__") || name.ends_with("__") || name.contains('.') {
+        return name.into();
+    }
+    // strip leading underscore
+    let class_name = class_name.strip_prefix(|c| c == '_').unwrap_or(class_name);
+    let mut ret = String::with_capacity(1 + class_name.len() + name.len());
+    ret.push('_');
+    ret.push_str(class_name);
+    ret.push_str(name);
+    ret.into()
+}

From dc0cada1f313d258b63209b2f7772323bb401953 Mon Sep 17 00:00:00 2001
From: kenkoooo <kenkou.n@gmail.com>
Date: Fri, 12 Feb 2021 21:56:59 +0900
Subject: [PATCH 190/459] Fix code format

---
 src/ir.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/ir.rs b/src/ir.rs
index 081eeb81..89cecc85 100644
--- a/src/ir.rs
+++ b/src/ir.rs
@@ -180,9 +180,12 @@ impl CodeInfo {
                 // we don't want to worry about Continue or Break, they use unwinding to jump to
                 // their targets and as such the stack size is taken care of in frame.rs by setting
                 // it back to the level it was at when SetupLoop was run
-                let jump_label = instr.label_arg().filter(
-                    |_| !matches!(instr, Instruction::Continue { .. } | Instruction::Break { .. }),
-                );
+                let jump_label = instr.label_arg().filter(|_| {
+                    !matches!(
+                        instr,
+                        Instruction::Continue { .. } | Instruction::Break { .. }
+                    )
+                });
                 if let Some(&target_block) = jump_label {
                     let effect = instr.stack_effect(true);
                     let target_depth = add_ui(depth, effect);

From d2b70cd5cdf1555ba4f0e743f7dbcc60273c9746 Mon Sep 17 00:00:00 2001
From: Padraic Fanning <fanninpm@miamioh.edu>
Date: Fri, 12 Feb 2021 16:23:06 -0500
Subject: [PATCH 191/459] Run rustfmt

---
 src/ir.rs | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/ir.rs b/src/ir.rs
index 081eeb81..89cecc85 100644
--- a/src/ir.rs
+++ b/src/ir.rs
@@ -180,9 +180,12 @@ impl CodeInfo {
                 // we don't want to worry about Continue or Break, they use unwinding to jump to
                 // their targets and as such the stack size is taken care of in frame.rs by setting
                 // it back to the level it was at when SetupLoop was run
-                let jump_label = instr.label_arg().filter(
-                    |_| !matches!(instr, Instruction::Continue { .. } | Instruction::Break { .. }),
-                );
+                let jump_label = instr.label_arg().filter(|_| {
+                    !matches!(
+                        instr,
+                        Instruction::Continue { .. } | Instruction::Break { .. }
+                    )
+                });
                 if let Some(&target_block) = jump_label {
                     let effect = instr.stack_effect(true);
                     let target_depth = add_ui(depth, effect);

From 5d1beacefa8cc4077dd6032df94177906e175bbb Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 3 Feb 2021 10:09:42 -0600
Subject: [PATCH 192/459] Handle splat args in bases/kwargs for __build_class__

---
 src/compile.rs | 180 ++++++++++++++++++++++++-------------------------
 1 file changed, 87 insertions(+), 93 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 4775dc18..626bec72 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1158,6 +1158,8 @@ impl Compiler {
     ) -> CompileResult<()> {
         self.prepare_decorators(decorator_list)?;
 
+        self.emit(Instruction::LoadBuildClass);
+
         let prev_ctx = self.ctx;
         self.ctx = CompileContext {
             func: FunctionContext::NoFunction,
@@ -1173,7 +1175,6 @@ impl Compiler {
             Some(qualified_name.clone()),
         );
 
-        self.emit(Instruction::LoadBuildClass);
         self.push_output(bytecode::CodeFlags::empty(), 0, 0, 0, name.to_owned());
 
         let (new_body, doc_str) = get_doc(body);
@@ -1241,35 +1242,7 @@ impl Compiler {
             value: qualified_name,
         });
 
-        for base in bases {
-            self.compile_expression(base)?;
-        }
-
-        if !keywords.is_empty() {
-            let mut kwarg_names = vec![];
-            for keyword in keywords {
-                if let Some(name) = &keyword.node.arg {
-                    kwarg_names.push(ConstantData::Str {
-                        value: name.to_owned(),
-                    });
-                } else {
-                    // This means **kwargs!
-                    panic!("name must be set");
-                }
-                self.compile_expression(&keyword.node.value)?;
-            }
-
-            self.emit_constant(ConstantData::Tuple {
-                elements: kwarg_names,
-            });
-            self.emit(Instruction::CallFunctionKeyword {
-                nargs: (2 + keywords.len() + bases.len()) as u32,
-            });
-        } else {
-            self.emit(Instruction::CallFunctionPositional {
-                nargs: (2 + bases.len()) as u32,
-            });
-        }
+        self.compile_call(2, bases, keywords)?;
 
         self.apply_decorators(decorator_list);
 
@@ -1821,7 +1794,10 @@ impl Compiler {
                 func,
                 args,
                 keywords,
-            } => self.compile_call(func, args, keywords)?,
+            } => {
+                self.compile_expression(func)?;
+                self.compile_call(0, args, keywords)?;
+            }
             BoolOp { op, values } => self.compile_bool_op(op, values)?,
             BinOp { left, op, right } => {
                 self.compile_expression(left)?;
@@ -1864,28 +1840,16 @@ impl Compiler {
                 self.emit_constant(compile_constant(value));
             }
             List { elts, .. } => {
-                let size = elts.len() as u32;
-                let must_unpack = self.gather_elements(elts)?;
-                self.emit(Instruction::BuildList {
-                    size,
-                    unpack: must_unpack,
-                });
+                let (size, unpack) = self.gather_elements(0, elts)?;
+                self.emit(Instruction::BuildList { size, unpack });
             }
             Tuple { elts, .. } => {
-                let size = elts.len() as u32;
-                let must_unpack = self.gather_elements(elts)?;
-                self.emit(Instruction::BuildTuple {
-                    size,
-                    unpack: must_unpack,
-                });
+                let (size, unpack) = self.gather_elements(0, elts)?;
+                self.emit(Instruction::BuildTuple { size, unpack });
             }
             Set { elts, .. } => {
-                let size = elts.len() as u32;
-                let must_unpack = self.gather_elements(elts)?;
-                self.emit(Instruction::BuildSet {
-                    size,
-                    unpack: must_unpack,
-                });
+                let (size, unpack) = self.gather_elements(0, elts)?;
+                self.emit(Instruction::BuildSet { size, unpack });
             }
             Dict { keys, values } => {
                 self.compile_dict(keys, values)?;
@@ -2139,81 +2103,111 @@ impl Compiler {
 
     fn compile_call(
         &mut self,
-        function: &ast::Expr,
+        additional_positional: u32,
         args: &[ast::Expr],
         keywords: &[ast::Keyword],
     ) -> CompileResult<()> {
-        self.compile_expression(function)?;
-        let count = (args.len() + keywords.len()) as u32;
+        let count = (args.len() + keywords.len()) as u32 + additional_positional;
 
         // Normal arguments:
-        let must_unpack = self.gather_elements(args)?;
+        let (size, unpack) = self.gather_elements(additional_positional, args)?;
         let has_double_star = keywords.iter().any(|k| k.node.arg.is_none());
 
-        if must_unpack || has_double_star {
+        if unpack || has_double_star {
             // Create a tuple with positional args:
-            self.emit(Instruction::BuildTuple {
-                size: args.len() as u32,
-                unpack: must_unpack,
-            });
+            self.emit(Instruction::BuildTuple { size, unpack });
 
             // Create an optional map with kw-args:
-            if !keywords.is_empty() {
+            let has_kwargs = !keywords.is_empty();
+            if has_kwargs {
                 self.compile_keywords(keywords)?;
-                self.emit(Instruction::CallFunctionEx { has_kwargs: true });
-            } else {
-                self.emit(Instruction::CallFunctionEx { has_kwargs: false });
             }
-        } else {
-            // Keyword arguments:
-            if !keywords.is_empty() {
-                let mut kwarg_names = vec![];
-                for keyword in keywords {
-                    if let Some(name) = &keyword.node.arg {
-                        kwarg_names.push(ConstantData::Str {
-                            value: name.to_owned(),
-                        });
-                    } else {
-                        // This means **kwargs!
-                        panic!("name must be set");
-                    }
-                    self.compile_expression(&keyword.node.value)?;
+            self.emit(Instruction::CallFunctionEx { has_kwargs });
+        } else if !keywords.is_empty() {
+            let mut kwarg_names = vec![];
+            for keyword in keywords {
+                if let Some(name) = &keyword.node.arg {
+                    kwarg_names.push(ConstantData::Str {
+                        value: name.to_owned(),
+                    });
+                } else {
+                    // This means **kwargs!
+                    panic!("name must be set");
                 }
-
-                self.emit_constant(ConstantData::Tuple {
-                    elements: kwarg_names,
-                });
-                self.emit(Instruction::CallFunctionKeyword { nargs: count });
-            } else {
-                self.emit(Instruction::CallFunctionPositional { nargs: count });
+                self.compile_expression(&keyword.node.value)?;
             }
+
+            self.emit_constant(ConstantData::Tuple {
+                elements: kwarg_names,
+            });
+            self.emit(Instruction::CallFunctionKeyword { nargs: count });
+        } else {
+            self.emit(Instruction::CallFunctionPositional { nargs: count });
         }
+
         Ok(())
     }
 
     // Given a vector of expr / star expr generate code which gives either
     // a list of expressions on the stack, or a list of tuples.
-    fn gather_elements(&mut self, elements: &[ast::Expr]) -> CompileResult<bool> {
+    fn gather_elements(
+        &mut self,
+        before: u32,
+        elements: &[ast::Expr],
+    ) -> CompileResult<(u32, bool)> {
         // First determine if we have starred elements:
         let has_stars = elements
             .iter()
             .any(|e| matches!(e.node, ast::ExprKind::Starred { .. }));
 
-        for element in elements {
-            if let ast::ExprKind::Starred { value, .. } = &element.node {
-                self.compile_expression(value)?;
-            } else {
-                self.compile_expression(element)?;
-                if has_stars {
+        let size = if has_stars {
+            let mut size = 0;
+
+            if before > 0 {
+                self.emit(Instruction::BuildTuple {
+                    size: before,
+                    unpack: false,
+                });
+                size += 1;
+            }
+
+            let groups = elements
+                .iter()
+                .map(|element| {
+                    if let ast::ExprKind::Starred { value, .. } = &element.node {
+                        (true, value.as_ref())
+                    } else {
+                        (false, element)
+                    }
+                })
+                .group_by(|(starred, _)| *starred);
+
+            for (starred, run) in &groups {
+                let mut run_size = 0;
+                for (_, value) in run {
+                    self.compile_expression(value)?;
+                    run_size += 1
+                }
+                if starred {
+                    size += run_size
+                } else {
                     self.emit(Instruction::BuildTuple {
-                        size: 1,
+                        size: run_size,
                         unpack: false,
                     });
+                    size += 1
                 }
             }
-        }
 
-        Ok(has_stars)
+            size
+        } else {
+            for element in elements {
+                self.compile_expression(element)?;
+            }
+            before + elements.len() as u32
+        };
+
+        Ok((size, has_stars))
     }
 
     fn compile_comprehension_element(&mut self, element: &ast::Expr) -> CompileResult<()> {

From 89c08246ba2988e20a9bba748ef29f297f907f1a Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Wed, 17 Feb 2021 17:11:39 -0600
Subject: [PATCH 193/459] Fix SetupAsyncWith stack level handling

---
 src/compile.rs                                | 17 +++++
 ...pile__tests__nested_double_async_with.snap | 76 +++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap

diff --git a/src/compile.rs b/src/compile.rs
index 626bec72..e25bd7c0 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -2563,6 +2563,23 @@ if True and False and False:
             "\
 if (True and False) or (False and True):
     pass
+"
+        ));
+    }
+
+    #[test]
+    fn test_nested_double_async_with() {
+        assert_dis_snapshot!(compile_exec(
+            "\
+for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):
+    with self.subTest(type=type(stop_exc)):
+        try:
+            async with woohoo():
+                raise stop_exc
+        except Exception as ex:
+            self.assertIs(ex, stop_exc)
+        else:
+            self.fail(f'{stop_exc} was suppressed')
 "
         ));
     }
diff --git a/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap b/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
new file mode 100644
index 00000000..b6f7d64f
--- /dev/null
+++ b/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
@@ -0,0 +1,76 @@
+---
+source: compiler/src/compile.rs
+expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n    with self.subTest(type=type(stop_exc)):\n        try:\n            async with woohoo():\n                raise stop_exc\n        except Exception as ex:\n            self.assertIs(ex, stop_exc)\n        else:\n            self.fail(f'{stop_exc} was suppressed')\n\")"
+---
+                 0 SetupLoop
+                 1 LoadNameAny          (0, StopIteration)
+                 2 LoadConst            ("spam")
+                 3 CallFunctionPositional (1)
+                 4 LoadNameAny          (1, StopAsyncIteration)
+                 5 LoadConst            ("ham")
+                 6 CallFunctionPositional (1)
+                 7 BuildTuple           (2, false)
+                 8 GetIter
+          >>     9 ForIter              (68)
+                10 StoreLocal           (2, stop_exc)
+                11 LoadNameAny          (3, self)
+                12 LoadAttr             (subTest)
+                13 LoadNameAny          (5, type)
+                14 LoadNameAny          (2, stop_exc)
+                15 CallFunctionPositional (1)
+                16 LoadConst            (("type"))
+                17 CallFunctionKeyword  (1)
+                18 SetupWith            (65)
+                19 Pop
+                20 SetupExcept          (40)
+                21 LoadNameAny          (6, woohoo)
+                22 CallFunctionPositional (0)
+                23 BeforeAsyncWith
+                24 GetAwaitable
+                25 LoadConst            (None)
+                26 YieldFrom
+                27 SetupAsyncWith       (33)
+                28 Pop
+                29 LoadNameAny          (2, stop_exc)
+                30 Raise                (Raise)
+                31 PopBlock
+                32 EnterFinally
+          >>    33 WithCleanupStart
+                34 GetAwaitable
+                35 LoadConst            (None)
+                36 YieldFrom
+                37 WithCleanupFinish
+                38 PopBlock
+                39 Jump                 (54)
+          >>    40 Duplicate
+                41 LoadNameAny          (7, Exception)
+                42 CompareOperation     (ExceptionMatch)
+                43 JumpIfFalse          (53)
+                44 StoreLocal           (8, ex)
+                45 LoadNameAny          (3, self)
+                46 LoadAttr             (assertIs)
+                47 LoadNameAny          (8, ex)
+                48 LoadNameAny          (2, stop_exc)
+                49 CallFunctionPositional (2)
+                50 Pop
+                51 PopException
+                52 Jump                 (63)
+          >>    53 Raise                (Reraise)
+          >>    54 LoadNameAny          (3, self)
+                55 LoadAttr             (fail)
+                56 LoadConst            ("")
+                57 LoadNameAny          (2, stop_exc)
+                58 FormatValue          (None)
+                59 LoadConst            (" was suppressed")
+                60 BuildString          (2)
+                61 CallFunctionPositional (1)
+                62 Pop
+          >>    63 PopBlock
+                64 EnterFinally
+          >>    65 WithCleanupStart
+                66 WithCleanupFinish
+                67 Jump                 (9)
+          >>    68 PopBlock
+                69 LoadConst            (None)
+                70 ReturnValue
+

From da0c85d2ac5c3cdd26127dae299aa0edc3493cb0 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Thu, 11 Mar 2021 09:58:25 -0600
Subject: [PATCH 194/459] Impl Eq+Hash for ConstantData, make
 CodeInfo.constants an IndexSet

---
 src/compile.rs | 7 +++----
 src/ir.rs      | 4 ++--
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index e25bd7c0..3dfd7731 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -157,7 +157,7 @@ impl Compiler {
 
             blocks: vec![ir::Block::default()],
             current_block: bytecode::Label(0),
-            constants: Vec::new(),
+            constants: IndexSet::default(),
             name_cache: IndexSet::default(),
             varname_cache: IndexSet::default(),
             cellvar_cache: IndexSet::default(),
@@ -235,7 +235,7 @@ impl Compiler {
 
             blocks: vec![ir::Block::default()],
             current_block: bytecode::Label(0),
-            constants: Vec::new(),
+            constants: IndexSet::default(),
             name_cache: IndexSet::default(),
             varname_cache: IndexSet::default(),
             cellvar_cache,
@@ -2377,8 +2377,7 @@ impl Compiler {
 
     fn emit_constant(&mut self, constant: ConstantData) {
         let info = self.current_codeinfo();
-        let idx = info.constants.len() as u32;
-        info.constants.push(constant);
+        let idx = info.constants.insert_full(constant).0 as u32;
         self.emit(Instruction::LoadConst { idx })
     }
 
diff --git a/src/ir.rs b/src/ir.rs
index 89cecc85..d2926bf6 100644
--- a/src/ir.rs
+++ b/src/ir.rs
@@ -37,7 +37,7 @@ pub struct CodeInfo {
 
     pub blocks: Vec<Block>,
     pub current_block: BlockIdx,
-    pub constants: Vec<ConstantData>,
+    pub constants: IndexSet<ConstantData>,
     pub name_cache: IndexSet<String>,
     pub varname_cache: IndexSet<String>,
     pub cellvar_cache: IndexSet<String>,
@@ -104,7 +104,7 @@ impl CodeInfo {
             max_stacksize,
             instructions: instructions.into_boxed_slice(),
             locations: locations.into_boxed_slice(),
-            constants: constants.into(),
+            constants: constants.into_iter().collect(),
             names: name_cache.into_iter().collect(),
             varnames: varname_cache.into_iter().collect(),
             cellvars: cellvar_cache.into_iter().collect(),

From ee0af406e25ffe4e63aa9065ad87d4d71e856388 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 15 Mar 2021 09:21:20 -0500
Subject: [PATCH 195/459] Add LoadMethod and CallMethod* opcodes

---
 src/compile.rs | 98 +++++++++++++++++++++++++++++++++++++-------------
 src/ir.rs      | 22 +++++++-----
 2 files changed, 87 insertions(+), 33 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 3dfd7731..8b2c7cf3 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -28,6 +28,28 @@ enum NameUsage {
     Delete,
 }
 
+enum CallType {
+    Positional { nargs: u32 },
+    Keyword { nargs: u32 },
+    Ex { has_kwargs: bool },
+}
+impl CallType {
+    fn normal_call(self) -> Instruction {
+        match self {
+            CallType::Positional { nargs } => Instruction::CallFunctionPositional { nargs },
+            CallType::Keyword { nargs } => Instruction::CallFunctionKeyword { nargs },
+            CallType::Ex { has_kwargs } => Instruction::CallFunctionEx { has_kwargs },
+        }
+    }
+    fn method_call(self) -> Instruction {
+        match self {
+            CallType::Positional { nargs } => Instruction::CallMethodPositional { nargs },
+            CallType::Keyword { nargs } => Instruction::CallMethodKeyword { nargs },
+            CallType::Ex { has_kwargs } => Instruction::CallMethodEx { has_kwargs },
+        }
+    }
+}
+
 /// Main structure holding the state of compilation.
 struct Compiler {
     code_stack: Vec<CodeInfo>,
@@ -667,14 +689,13 @@ impl Compiler {
                     self.switch_to_block(after_block);
                 }
             }
-            Break => match self.ctx.loop_data {
-                Some((_, end)) => {
-                    self.emit(Instruction::Break { target: end });
-                }
-                None => {
-                    return Err(self.error_loc(CompileErrorType::InvalidBreak, statement.location))
+            Break => {
+                if self.ctx.loop_data.is_some() {
+                    self.emit(Instruction::Break);
+                } else {
+                    return Err(self.error_loc(CompileErrorType::InvalidBreak, statement.location));
                 }
-            },
+            }
             Continue => match self.ctx.loop_data {
                 Some((start, _)) => {
                     self.emit(Instruction::Continue { target: start });
@@ -1242,7 +1263,8 @@ impl Compiler {
             value: qualified_name,
         });
 
-        self.compile_call(2, bases, keywords)?;
+        let call = self.compile_call_inner(2, bases, keywords)?;
+        self.emit(call.normal_call());
 
         self.apply_decorators(decorator_list);
 
@@ -1271,7 +1293,9 @@ impl Compiler {
         let else_block = self.new_block();
         let after_block = self.new_block();
 
-        self.emit(Instruction::SetupLoop);
+        self.emit(Instruction::SetupLoop {
+            break_target: after_block,
+        });
         self.switch_to_block(while_block);
 
         self.compile_jump_if(test, false, else_block)?;
@@ -1360,7 +1384,9 @@ impl Compiler {
         let else_block = self.new_block();
         let after_block = self.new_block();
 
-        self.emit(Instruction::SetupLoop);
+        self.emit(Instruction::SetupLoop {
+            break_target: after_block,
+        });
 
         // The thing iterated:
         self.compile_expression(iter)?;
@@ -1794,10 +1820,7 @@ impl Compiler {
                 func,
                 args,
                 keywords,
-            } => {
-                self.compile_expression(func)?;
-                self.compile_call(0, args, keywords)?;
-            }
+            } => self.compile_call(func, args, keywords)?,
             BoolOp { op, values } => self.compile_bool_op(op, values)?,
             BinOp { left, op, right } => {
                 self.compile_expression(left)?;
@@ -2103,17 +2126,41 @@ impl Compiler {
 
     fn compile_call(
         &mut self,
-        additional_positional: u32,
+        func: &ast::Expr,
         args: &[ast::Expr],
         keywords: &[ast::Keyword],
     ) -> CompileResult<()> {
+        let method = if let ast::ExprKind::Attribute { value, attr, .. } = &func.node {
+            self.compile_expression(value)?;
+            let idx = self.name(attr);
+            self.emit(Instruction::LoadMethod { idx });
+            true
+        } else {
+            self.compile_expression(func)?;
+            false
+        };
+        let call = self.compile_call_inner(0, args, keywords)?;
+        self.emit(if method {
+            call.method_call()
+        } else {
+            call.normal_call()
+        });
+        Ok(())
+    }
+
+    fn compile_call_inner(
+        &mut self,
+        additional_positional: u32,
+        args: &[ast::Expr],
+        keywords: &[ast::Keyword],
+    ) -> CompileResult<CallType> {
         let count = (args.len() + keywords.len()) as u32 + additional_positional;
 
         // Normal arguments:
         let (size, unpack) = self.gather_elements(additional_positional, args)?;
         let has_double_star = keywords.iter().any(|k| k.node.arg.is_none());
 
-        if unpack || has_double_star {
+        let call = if unpack || has_double_star {
             // Create a tuple with positional args:
             self.emit(Instruction::BuildTuple { size, unpack });
 
@@ -2122,7 +2169,7 @@ impl Compiler {
             if has_kwargs {
                 self.compile_keywords(keywords)?;
             }
-            self.emit(Instruction::CallFunctionEx { has_kwargs });
+            CallType::Ex { has_kwargs }
         } else if !keywords.is_empty() {
             let mut kwarg_names = vec![];
             for keyword in keywords {
@@ -2140,12 +2187,12 @@ impl Compiler {
             self.emit_constant(ConstantData::Tuple {
                 elements: kwarg_names,
             });
-            self.emit(Instruction::CallFunctionKeyword { nargs: count });
+            CallType::Keyword { nargs: count }
         } else {
-            self.emit(Instruction::CallFunctionPositional { nargs: count });
-        }
+            CallType::Positional { nargs: count }
+        };
 
-        Ok(())
+        Ok(call)
     }
 
     // Given a vector of expr / star expr generate code which gives either
@@ -2262,8 +2309,13 @@ impl Compiler {
                 unimplemented!("async for comprehensions");
             }
 
+            let loop_block = self.new_block();
+            let after_block = self.new_block();
+
             // Setup for loop:
-            self.emit(Instruction::SetupLoop);
+            self.emit(Instruction::SetupLoop {
+                break_target: after_block,
+            });
 
             if loop_labels.is_empty() {
                 // Load iterator onto stack (passed as first argument):
@@ -2276,8 +2328,6 @@ impl Compiler {
                 self.emit(Instruction::GetIter);
             }
 
-            let loop_block = self.new_block();
-            let after_block = self.new_block();
             loop_labels.push((loop_block, after_block));
 
             self.switch_to_block(loop_block);
diff --git a/src/ir.rs b/src/ir.rs
index d2926bf6..3c7ba9ff 100644
--- a/src/ir.rs
+++ b/src/ir.rs
@@ -167,25 +167,29 @@ impl CodeInfo {
         let mut startdepths = vec![u32::MAX; self.blocks.len()];
         startdepths[0] = 0;
         stack.push(Label(0));
+        let debug = false;
         'process_blocks: while let Some(block) = stack.pop() {
             let mut depth = startdepths[block.0 as usize];
+            if debug {
+                eprintln!("===BLOCK {}===", block.0);
+            }
             let block = &self.blocks[block.0 as usize];
             for i in &block.instructions {
                 let instr = &i.instr;
                 let effect = instr.stack_effect(false);
                 let new_depth = add_ui(depth, effect);
+                if debug {
+                    eprintln!("{:?}: {:+}, {:+} = {}", instr, effect, depth, new_depth);
+                }
                 if new_depth > maxdepth {
                     maxdepth = new_depth
                 }
-                // we don't want to worry about Continue or Break, they use unwinding to jump to
-                // their targets and as such the stack size is taken care of in frame.rs by setting
+                // we don't want to worry about Continue, it uses unwinding to jump to
+                // its targets and as such the stack size is taken care of in frame.rs by setting
                 // it back to the level it was at when SetupLoop was run
-                let jump_label = instr.label_arg().filter(|_| {
-                    !matches!(
-                        instr,
-                        Instruction::Continue { .. } | Instruction::Break { .. }
-                    )
-                });
+                let jump_label = instr
+                    .label_arg()
+                    .filter(|_| !matches!(instr, Instruction::Continue { .. }));
                 if let Some(&target_block) = jump_label {
                     let effect = instr.stack_effect(true);
                     let target_depth = add_ui(depth, effect);
@@ -215,7 +219,7 @@ fn stackdepth_push(stack: &mut Vec<Label>, startdepths: &mut [u32], target: Labe
 
 fn add_ui(a: u32, b: i32) -> u32 {
     if b < 0 {
-        a - b.abs() as u32
+        a - b.wrapping_abs() as u32
     } else {
         a + b as u32
     }

From ee26bc84ecb4f472147d7deb643c5f06f75b9888 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Tue, 23 Mar 2021 12:39:48 -0500
Subject: [PATCH 196/459] Upgrade dependencies

---
 Cargo.toml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index d302365e..891b26df 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,14 +9,14 @@ edition = "2018"
 
 [dependencies]
 indexmap = "1.0"
-itertools = "0.9"
+itertools = "0.10.0"
 rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
 rustpython-ast = { path = "../ast" }
-num-complex = { version = "0.3", features = ["serde"] }
+num-complex = { version = "0.4.0", features = ["serde"] }
 num-traits = "0.2"
 log = "0.4"
-ahash = "0.6"
+ahash = "0.7.2"
 
 [dev-dependencies]
 rustpython-parser = { path = "../parser" }
-insta = "1.5"
+insta = "1.7.0"

From b35fe39531389eb1daaa9472cff4a0f5ac8dbf13 Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Sun, 4 Apr 2021 18:03:19 -0500
Subject: [PATCH 197/459] Add an ast optimizer

---
 porcelain/src/lib.rs | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/porcelain/src/lib.rs b/porcelain/src/lib.rs
index c4bd72a6..8308cae0 100644
--- a/porcelain/src/lib.rs
+++ b/porcelain/src/lib.rs
@@ -1,6 +1,7 @@
 use rustpython_bytecode::CodeObject;
 use rustpython_compiler_core::{compile, symboltable};
-use rustpython_parser::{ast::Location, parser};
+use rustpython_parser::ast::{fold::Fold, ConstantOptimizer, Location};
+use rustpython_parser::parser;
 use std::fmt;
 
 pub use compile::{CompileOpts, Mode};
@@ -80,10 +81,15 @@ pub fn compile(
         compile::Mode::Eval => parser::Mode::Expression,
         compile::Mode::Single => parser::Mode::Interactive,
     };
-    let ast = match parser::parse(source, mode) {
+    let mut ast = match parser::parse(source, mode) {
         Ok(x) => x,
         Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
     };
+    if opts.optimize > 0 {
+        ast = ConstantOptimizer::new()
+            .fold_mod(ast)
+            .unwrap_or_else(|e| match e {});
+    }
     compile::compile_top(&ast, source_path, opts).map_err(|e| CompileError::from_compile(e, source))
 }
 

From ab1dd4e2a4803fbdee5fdbe8b0120d43fa0b997b Mon Sep 17 00:00:00 2001
From: Noah <33094578+coolreader18@users.noreply.github.com>
Date: Tue, 6 Apr 2021 21:00:06 -0500
Subject: [PATCH 198/459] Fix async for compilation

---
 src/compile.rs | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 8b2c7cf3..0d65bacf 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1391,22 +1391,18 @@ impl Compiler {
         // The thing iterated:
         self.compile_expression(iter)?;
 
-        let check_asynciter_block = if is_async {
-            let check_asynciter_block = self.new_block();
-
+        if is_async {
             self.emit(Instruction::GetAIter);
 
             self.switch_to_block(for_block);
             self.emit(Instruction::SetupExcept {
-                handler: check_asynciter_block,
+                handler: else_block,
             });
             self.emit(Instruction::GetANext);
             self.emit_constant(ConstantData::None);
             self.emit(Instruction::YieldFrom);
             self.compile_store(target)?;
             self.emit(Instruction::PopBlock);
-
-            Some(check_asynciter_block)
         } else {
             // Retrieve Iterator
             self.emit(Instruction::GetIter);
@@ -1416,7 +1412,6 @@ impl Compiler {
 
             // Start of loop iteration, set targets:
             self.compile_store(target)?;
-            None
         };
 
         let was_in_loop = self.ctx.loop_data;
@@ -1425,12 +1420,10 @@ impl Compiler {
         self.ctx.loop_data = was_in_loop;
         self.emit(Instruction::Jump { target: for_block });
 
-        if let Some(check_asynciter_block) = check_asynciter_block {
-            self.switch_to_block(check_asynciter_block);
+        self.switch_to_block(else_block);
+        if is_async {
             self.emit(Instruction::EndAsyncFor);
         }
-
-        self.switch_to_block(else_block);
         self.emit(Instruction::PopBlock);
         self.compile_statements(orelse)?;
 

From 5d639cc8deda327381aa270e2877c8b27a858385 Mon Sep 17 00:00:00 2001
From: Padraic Fanning <fanninpm@miamioh.edu>
Date: Fri, 7 May 2021 16:57:38 -0400
Subject: [PATCH 199/459] Fix manual_flatten Clippy warning

---
 src/symboltable.rs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 44c6df90..8eaa0958 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -1074,10 +1074,8 @@ impl SymbolTableBuilder {
     ) -> SymbolTableResult {
         // Evaluate eventual default parameters:
         self.scan_expressions(&args.defaults, ExpressionContext::Load)?;
-        for kw_default in &args.kw_defaults {
-            if let Some(expression) = kw_default {
-                self.scan_expression(&expression, ExpressionContext::Load)?;
-            }
+        for expression in args.kw_defaults.iter().flatten() {
+            self.scan_expression(&expression, ExpressionContext::Load)?;
         }
 
         // Annotations are scanned in outer scope:

From 9b0a82aed359d9bd834ec62dbaefab4d903c65ba Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Fri, 30 Jul 2021 02:30:59 +0900
Subject: [PATCH 200/459] Fix unnessessary borrow

---
 src/compile.rs     | 8 ++++----
 src/symboltable.rs | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 0d65bacf..db33c5b8 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -348,7 +348,7 @@ impl Compiler {
                     self.emit(Instruction::PrintExpr);
                 }
             } else {
-                self.compile_statement(&statement)?;
+                self.compile_statement(statement)?;
             }
         }
 
@@ -470,7 +470,7 @@ impl Compiler {
             // we do this here because `from __future__` still executes that `from` statement at runtime,
             // we still need to compile the ImportFrom down below
             ImportFrom { module, names, .. } if module.as_deref() == Some("__future__") => {
-                self.compile_future_features(&names)?
+                self.compile_future_features(names)?
             }
             // if we find any other statement, stop accepting future statements
             _ => self.done_with_future_stmts = true,
@@ -1068,7 +1068,7 @@ impl Compiler {
                 self.emit_constant(ConstantData::Str {
                     value: self.mangle(&arg.node.arg).into_owned(),
                 });
-                self.compile_expression(&annotation)?;
+                self.compile_expression(annotation)?;
                 num_annotations += 1;
             }
         }
@@ -1159,7 +1159,7 @@ impl Compiler {
                     orelse,
                     finalbody,
                     ..
-                } => self.find_ann(&body) || self.find_ann(orelse) || self.find_ann(finalbody),
+                } => self.find_ann(body) || self.find_ann(orelse) || self.find_ann(finalbody),
                 _ => false,
             };
             if res {
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 8eaa0958..10c13ba5 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -617,7 +617,7 @@ impl SymbolTableBuilder {
 
     fn scan_parameter_annotation(&mut self, parameter: &ast::Arg) -> SymbolTableResult {
         if let Some(annotation) = &parameter.node.annotation {
-            self.scan_expression(&annotation, ExpressionContext::Load)?;
+            self.scan_expression(annotation, ExpressionContext::Load)?;
         }
         Ok(())
     }
@@ -1075,7 +1075,7 @@ impl SymbolTableBuilder {
         // Evaluate eventual default parameters:
         self.scan_expressions(&args.defaults, ExpressionContext::Load)?;
         for expression in args.kw_defaults.iter().flatten() {
-            self.scan_expression(&expression, ExpressionContext::Load)?;
+            self.scan_expression(expression, ExpressionContext::Load)?;
         }
 
         // Annotations are scanned in outer scope:

From 0383dede8178d9441b7c0fe0f7f8063f0a0cbad7 Mon Sep 17 00:00:00 2001
From: snowapril <sinjihng@gmail.com>
Date: Tue, 17 Aug 2021 19:58:37 +0900
Subject: [PATCH 201/459] compiler: impl syntax error detect for global kind

This commit fixes `test_global_parap_err_first` test in
[test_syntax.py](https://github.com/RustPython/RustPython/blob/master/Lib/test/test_syntax.py#L678) and implement the other syntax errors detection as cpython 3.8 provided for global symbol.

Below syntax error conditions are added.
* name {} is parameter and global
```python
>>>>> def test(a):
.....   global a
.....
SyntaxError: name 'a' is parameter and global at line 2 column 2
        global a
```
* annotated name {} can't be global
```python
>>>>> def test():
.....   a: int
.....   global a
.....
SyntaxError: annotated name 'a' can't be global at line 3 column 2
        global a
```
* name {} is assigned to before global description
```python
>>>>> a = 100
>>>>> def test():
.....   a = 10
.....   global a
.....
SyntaxError: name 'a' is assigned to before global declaration at line 3 column 2
        global a
```
* name {} is used prior to global declaration
```python
>>>>> a = 10
>>>>> def test():
.....   print(a)
.....   global a
.....
SyntaxError: name 'a' is used prior to global declaration at line 3 column 2
        global a
```

Signed-off-by: snowapril <sinjihng@gmail.com>
---
 src/symboltable.rs | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 10c13ba5..8dc120e2 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -1120,12 +1120,33 @@ impl SymbolTableBuilder {
             // Role already set..
             match role {
                 SymbolUsage::Global => {
-                    if !symbol.is_global() {
+                    if symbol.is_parameter {
+                        return Err(SymbolTableError {
+                            error: format!("name '{}' is parameter and global", name),
+                            location,
+                        });
+                    }
+                    if symbol.is_referenced {
                         return Err(SymbolTableError {
                             error: format!("name '{}' is used prior to global declaration", name),
                             location,
                         });
                     }
+                    if symbol.is_annotated {
+                        return Err(SymbolTableError {
+                            error: format!("annotated name '{}' can't be global", name),
+                            location,
+                        });
+                    }
+                    if symbol.is_assigned {
+                        return Err(SymbolTableError {
+                            error: format!(
+                                "name '{}' is assigned to before global declaration",
+                                name
+                            ),
+                            location,
+                        });
+                    }
                 }
                 SymbolUsage::Nonlocal => {
                     if symbol.is_parameter {

From 416e2da7f6df81d44b991d3e3648872eff422bbd Mon Sep 17 00:00:00 2001
From: snowapril <sinjihng@gmail.com>
Date: Wed, 18 Aug 2021 00:35:14 +0900
Subject: [PATCH 202/459] compiler: add is_global() check at top of errors

add `!symbol.is_global()` condition check before entrying global symbol
errors

This change will decrease hot path cost.

Signed-off-by: snowapril <sinjihng@gmail.com>
---
 src/symboltable.rs | 57 +++++++++++++++++++++++++---------------------
 1 file changed, 31 insertions(+), 26 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 8dc120e2..1bdd1a2d 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -1120,32 +1120,37 @@ impl SymbolTableBuilder {
             // Role already set..
             match role {
                 SymbolUsage::Global => {
-                    if symbol.is_parameter {
-                        return Err(SymbolTableError {
-                            error: format!("name '{}' is parameter and global", name),
-                            location,
-                        });
-                    }
-                    if symbol.is_referenced {
-                        return Err(SymbolTableError {
-                            error: format!("name '{}' is used prior to global declaration", name),
-                            location,
-                        });
-                    }
-                    if symbol.is_annotated {
-                        return Err(SymbolTableError {
-                            error: format!("annotated name '{}' can't be global", name),
-                            location,
-                        });
-                    }
-                    if symbol.is_assigned {
-                        return Err(SymbolTableError {
-                            error: format!(
-                                "name '{}' is assigned to before global declaration",
-                                name
-                            ),
-                            location,
-                        });
+                    if !symbol.is_global() {
+                        if symbol.is_parameter {
+                            return Err(SymbolTableError {
+                                error: format!("name '{}' is parameter and global", name),
+                                location,
+                            });
+                        }
+                        if symbol.is_referenced {
+                            return Err(SymbolTableError {
+                                error: format!(
+                                    "name '{}' is used prior to global declaration",
+                                    name
+                                ),
+                                location,
+                            });
+                        }
+                        if symbol.is_annotated {
+                            return Err(SymbolTableError {
+                                error: format!("annotated name '{}' can't be global", name),
+                                location,
+                            });
+                        }
+                        if symbol.is_assigned {
+                            return Err(SymbolTableError {
+                                error: format!(
+                                    "name '{}' is assigned to before global declaration",
+                                    name
+                                ),
+                                location,
+                            });
+                        }
                     }
                 }
                 SymbolUsage::Nonlocal => {

From 3289b9ccb4d9c9c5f603d3507a6257bda5c784f6 Mon Sep 17 00:00:00 2001
From: Nikita Sobolev <mail@sobolevn.me>
Date: Sun, 22 Aug 2021 06:42:13 +0300
Subject: [PATCH 203/459] Fixes typos in all other files

---
 src/symboltable.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 1bdd1a2d..f5cdde21 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -111,7 +111,7 @@ pub struct Symbol {
     // this is required to correct the scope in the analysis.
     pub is_assign_namedexpr_in_comprehension: bool,
 
-    // inidicates that the symbol is used a bound iterator variable. We distinguish this case
+    // indicates that the symbol is used a bound iterator variable. We distinguish this case
     // from normal assignment to detect unallowed re-assignment to iterator variables.
     pub is_iter: bool,
 
@@ -470,7 +470,7 @@ impl SymbolTableAnalyzer {
             SymbolTableType::Function => {
                 if let Some(parent_symbol) = symbols.get_mut(&symbol.name) {
                     if let SymbolScope::Unknown = parent_symbol.scope {
-                        // this information is new, as the asignment is done in inner scope
+                        // this information is new, as the assignment is done in inner scope
                         parent_symbol.is_assigned = true;
                     }
 
@@ -504,7 +504,7 @@ impl SymbolTableAnalyzer {
                     None => {
                         // extend the scope of the inner symbol
                         // as we are in a nested comprehension, we expect that the symbol is needed
-                        // ouside, too, and set it therefore to non-local scope. I.e., we expect to
+                        // outside, too, and set it therefore to non-local scope. I.e., we expect to
                         // find a definition on a higher level
                         let mut cloned_sym = symbol.clone();
                         cloned_sym.scope = SymbolScope::Free;
@@ -989,7 +989,7 @@ impl SymbolTableBuilder {
             }
 
             NamedExpr { target, value } => {
-                // named expressions are not allowed in the definiton of
+                // named expressions are not allowed in the definition of
                 // comprehension iterator definitions
                 if let ExpressionContext::IterDefinitionExp = context {
                     return Err(SymbolTableError {

From ca37219affe58bc3372a9292b685ba5f0d446fd9 Mon Sep 17 00:00:00 2001
From: Lee Dogeon <dev.moreal@gmail.com>
Date: Fri, 27 Aug 2021 23:07:10 +0900
Subject: [PATCH 204/459] Fix `__repr__`, `__qualname__` (#2977)

---
 src/compile.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index db33c5b8..fb09e530 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1021,8 +1021,8 @@ impl Compiler {
         };
 
         let qualified_name = self.create_qualified_name(name, "");
-        let old_qualified_path = self.current_qualified_path.take();
-        self.current_qualified_path = Some(self.create_qualified_name(name, ".<locals>"));
+        let old_qualified_path = self.current_qualified_path.replace(qualified_name.clone());
+        self.current_qualified_path = Some(self.create_qualified_name("<locals>", ""));
 
         let (body, doc_str) = get_doc(body);
 

From 88321d93199e3b9b84d29ebf46f5ece980122437 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Fri, 27 Aug 2021 23:19:50 +0900
Subject: [PATCH 205/459] clean up Compiler::create_qualified_name

---
 src/compile.rs | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index fb09e530..259d7666 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1020,9 +1020,10 @@ impl Compiler {
             },
         };
 
-        let qualified_name = self.create_qualified_name(name, "");
-        let old_qualified_path = self.current_qualified_path.replace(qualified_name.clone());
-        self.current_qualified_path = Some(self.create_qualified_name("<locals>", ""));
+        let old_qualified_path = self.current_qualified_path.clone();
+        self.push_qualified_name(name);
+        let qualified_name = self.current_qualified_path.clone().unwrap();
+        self.push_qualified_name("<locals>");
 
         let (body, doc_str) = get_doc(body);
 
@@ -1190,11 +1191,9 @@ impl Compiler {
 
         let prev_class_name = std::mem::replace(&mut self.class_name, Some(name.to_owned()));
 
-        let qualified_name = self.create_qualified_name(name, "");
-        let old_qualified_path = std::mem::replace(
-            &mut self.current_qualified_path,
-            Some(qualified_name.clone()),
-        );
+        let old_qualified_path = self.current_qualified_path.clone();
+        self.push_qualified_name(name);
+        let qualified_name = self.current_qualified_path.clone().unwrap();
 
         self.push_output(bytecode::CodeFlags::empty(), 0, 0, 0, name.to_owned());
 
@@ -2466,11 +2465,12 @@ impl Compiler {
         self.current_source_location.row()
     }
 
-    fn create_qualified_name(&self, name: &str, suffix: &str) -> String {
-        if let Some(ref qualified_path) = self.current_qualified_path {
-            format!("{}.{}{}", qualified_path, name, suffix)
+    fn push_qualified_name(&mut self, name: &str) {
+        if let Some(ref mut qualified_path) = self.current_qualified_path {
+            qualified_path.push('.');
+            qualified_path.push_str(name);
         } else {
-            format!("{}{}", name, suffix)
+            self.current_qualified_path = Some(name.to_owned());
         }
     }
 

From acd73d21fe2dd13604eb449b158f46e5e72ffa2d Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Fri, 27 Aug 2021 23:45:47 +0900
Subject: [PATCH 206/459] Compiler::qualified_name as vec

---
 src/compile.rs | 30 ++++++++++++------------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 259d7666..beded10c 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -56,7 +56,7 @@ struct Compiler {
     symbol_table_stack: Vec<SymbolTable>,
     source_path: String,
     current_source_location: ast::Location,
-    current_qualified_path: Option<String>,
+    qualified_path: Vec<String>,
     done_with_future_stmts: bool,
     ctx: CompileContext,
     class_name: Option<String>,
@@ -190,7 +190,7 @@ impl Compiler {
             symbol_table_stack: Vec::new(),
             source_path,
             current_source_location: ast::Location::default(),
-            current_qualified_path: None,
+            qualified_path: Vec::new(),
             done_with_future_stmts: false,
             ctx: CompileContext {
                 loop_data: None,
@@ -1020,10 +1020,9 @@ impl Compiler {
             },
         };
 
-        let old_qualified_path = self.current_qualified_path.clone();
-        self.push_qualified_name(name);
-        let qualified_name = self.current_qualified_path.clone().unwrap();
-        self.push_qualified_name("<locals>");
+        self.push_qualified_path(name);
+        let qualified_name = self.qualified_path.join(".");
+        self.push_qualified_path("<locals>");
 
         let (body, doc_str) = get_doc(body);
 
@@ -1041,7 +1040,8 @@ impl Compiler {
         }
 
         let code = self.pop_code_object();
-        self.current_qualified_path = old_qualified_path;
+        self.qualified_path.pop();
+        self.qualified_path.pop();
         self.ctx = prev_ctx;
 
         // Prepare type annotations:
@@ -1191,9 +1191,8 @@ impl Compiler {
 
         let prev_class_name = std::mem::replace(&mut self.class_name, Some(name.to_owned()));
 
-        let old_qualified_path = self.current_qualified_path.clone();
-        self.push_qualified_name(name);
-        let qualified_name = self.current_qualified_path.clone().unwrap();
+        self.push_qualified_path(name);
+        let qualified_name = self.qualified_path.join(".");
 
         self.push_output(bytecode::CodeFlags::empty(), 0, 0, 0, name.to_owned());
 
@@ -1239,7 +1238,7 @@ impl Compiler {
         let code = self.pop_code_object();
 
         self.class_name = prev_class_name;
-        self.current_qualified_path = old_qualified_path;
+        self.qualified_path.pop();
         self.ctx = prev_ctx;
 
         let mut funcflags = bytecode::MakeFunctionFlags::empty();
@@ -2465,13 +2464,8 @@ impl Compiler {
         self.current_source_location.row()
     }
 
-    fn push_qualified_name(&mut self, name: &str) {
-        if let Some(ref mut qualified_path) = self.current_qualified_path {
-            qualified_path.push('.');
-            qualified_path.push_str(name);
-        } else {
-            self.current_qualified_path = Some(name.to_owned());
-        }
+    fn push_qualified_path(&mut self, name: &str) {
+        self.qualified_path.push(name.to_owned());
     }
 
     fn mark_generator(&mut self) {

From 94e77ef8a63a8e427794dd978a620a798c7d3fe6 Mon Sep 17 00:00:00 2001
From: HyeockJinKim <kherootz@gmail.com>
Date: Fri, 27 Aug 2021 23:27:11 +0900
Subject: [PATCH 207/459] Raise syntax error when saving a value to __debug__.

---
 src/compile.rs | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index fb09e530..3ac7f07d 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -33,6 +33,7 @@ enum CallType {
     Keyword { nargs: u32 },
     Ex { has_kwargs: bool },
 }
+
 impl CallType {
     fn normal_call(self) -> Instruction {
         match self {
@@ -69,6 +70,7 @@ pub struct CompileOpts {
     /// not emit assert statements
     pub optimize: u8,
 }
+
 impl Default for CompileOpts {
     fn default() -> Self {
         CompileOpts { optimize: 0 }
@@ -383,8 +385,14 @@ impl Compiler {
         self.compile_name(name, NameUsage::Load)
     }
 
-    fn store_name(&mut self, name: &str) {
-        self.compile_name(name, NameUsage::Store)
+    fn store_name(&mut self, name: &str) -> CompileResult<()> {
+        if name.eq("__debug__") {
+            return Err(self.error(CompileErrorType::SyntaxError(
+                "cannot assign to __debug__".to_owned(),
+            )));
+        }
+        self.compile_name(name, NameUsage::Store);
+        Ok(())
     }
 
     fn mangle<'a>(&self, name: &'a str) -> Cow<'a, str> {
@@ -491,9 +499,9 @@ impl Compiler {
                             let idx = self.name(part);
                             self.emit(Instruction::LoadAttr { idx });
                         }
-                        self.store_name(alias);
+                        self.store_name(alias)?;
                     } else {
-                        self.store_name(name.name.split('.').next().unwrap());
+                        self.store_name(name.name.split('.').next().unwrap())?;
                     }
                 }
             }
@@ -549,9 +557,9 @@ impl Compiler {
 
                         // Store module under proper name:
                         if let Some(alias) = &name.asname {
-                            self.store_name(alias);
+                            self.store_name(alias)?;
                         } else {
-                            self.store_name(&name.name);
+                            self.store_name(&name.name)?;
                         }
                     }
 
@@ -933,7 +941,7 @@ impl Compiler {
 
                 // We have a match, store in name (except x as y)
                 if let Some(alias) = name {
-                    self.store_name(alias);
+                    self.store_name(alias)?;
                 } else {
                     // Drop exception from top of stack:
                     self.emit(Instruction::Pop);
@@ -1104,7 +1112,7 @@ impl Compiler {
 
         self.apply_decorators(decorator_list);
 
-        self.store_name(name);
+        self.store_name(name)?;
 
         Ok(())
     }
@@ -1268,7 +1276,7 @@ impl Compiler {
 
         self.apply_decorators(decorator_list);
 
-        self.store_name(name);
+        self.store_name(name)?;
         Ok(())
     }
 
@@ -1554,7 +1562,7 @@ impl Compiler {
     fn compile_store(&mut self, target: &ast::Expr) -> CompileResult<()> {
         match &target.node {
             ast::ExprKind::Name { id, .. } => {
-                self.store_name(id);
+                self.store_name(id)?;
             }
             ast::ExprKind::Subscript { value, slice, .. } => {
                 self.compile_expression(value)?;
@@ -1610,7 +1618,7 @@ impl Compiler {
                         "starred assignment target must be in a list or tuple".to_owned(),
                     ),
                     _ => CompileErrorType::Assign(target.node.name()),
-                }))
+                }));
             }
         }
 
@@ -1909,10 +1917,10 @@ impl Compiler {
             YieldFrom { value } => {
                 match self.ctx.func {
                     FunctionContext::NoFunction => {
-                        return Err(self.error(CompileErrorType::InvalidYieldFrom))
+                        return Err(self.error(CompileErrorType::InvalidYieldFrom));
                     }
                     FunctionContext::AsyncFunction => {
-                        return Err(self.error(CompileErrorType::AsyncYieldFrom))
+                        return Err(self.error(CompileErrorType::AsyncYieldFrom));
                     }
                     FunctionContext::Function => {}
                 }

From 91ecc3df9a71954c56003eac4fce4f78f3598314 Mon Sep 17 00:00:00 2001
From: HyeockJinKim <kherootz@gmail.com>
Date: Sun, 5 Sep 2021 17:50:08 +0900
Subject: [PATCH 208/459] remove unnecessary mut in compiler.rs

---
 src/compile.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index beded10c..f78e6417 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -2460,7 +2460,7 @@ impl Compiler {
         self.current_source_location = location;
     }
 
-    fn get_source_line_number(&mut self) -> usize {
+    fn get_source_line_number(&self) -> usize {
         self.current_source_location.row()
     }
 

From ae122c6478fb0da87018ebee4560d87cba692401 Mon Sep 17 00:00:00 2001
From: Padraic Fanning <fanninpm@miamioh.edu>
Date: Wed, 15 Sep 2021 17:54:54 -0400
Subject: [PATCH 209/459] Replace `master` with `main`

Fixes #3067
---
 src/compile.rs     | 2 +-
 src/lib.rs         | 2 +-
 src/symboltable.rs | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 4dd9e397..0ee81d35 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -2,7 +2,7 @@
 //! Take an AST and transform it into bytecode
 //!
 //! Inspirational code:
-//!   https://github.com/python/cpython/blob/master/Python/compile.c
+//!   https://github.com/python/cpython/blob/main/Python/compile.c
 //!   https://github.com/micropython/micropython/blob/master/py/compile.c
 
 use crate::ir::{self, CodeInfo};
diff --git a/src/lib.rs b/src/lib.rs
index 3116547d..3e3aca8f 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,5 +1,5 @@
 //! Compile a Python AST or source code into bytecode consumable by RustPython.
-#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/master/logo.png")]
+#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
 #![doc(html_root_url = "https://docs.rs/rustpython-compiler/")]
 
 #[macro_use]
diff --git a/src/symboltable.rs b/src/symboltable.rs
index f5cdde21..c0fd9a22 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -4,7 +4,7 @@ This ensures that global and nonlocal keywords are picked up.
 Then the compiler can use the symbol table to generate proper
 load and store instructions for names.
 
-Inspirational file: https://github.com/python/cpython/blob/master/Python/symtable.c
+Inspirational file: https://github.com/python/cpython/blob/main/Python/symtable.c
 */
 
 use crate::error::{CompileError, CompileErrorType};
@@ -196,7 +196,7 @@ impl std::fmt::Debug for SymbolTable {
 }
 
 /* Perform some sort of analysis on nonlocals, globals etc..
-  See also: https://github.com/python/cpython/blob/master/Python/symtable.c#L410
+  See also: https://github.com/python/cpython/blob/main/Python/symtable.c#L410
 */
 fn analyze_symbol_table(symbol_table: &mut SymbolTable) -> SymbolTableResult {
     let mut analyzer = SymbolTableAnalyzer::default();
@@ -760,7 +760,7 @@ impl SymbolTableBuilder {
                 value,
                 simple,
             } => {
-                // https://github.com/python/cpython/blob/master/Python/symtable.c#L1233
+                // https://github.com/python/cpython/blob/main/Python/symtable.c#L1233
                 match &target.node {
                     ast::ExprKind::Name { id, .. } if *simple => {
                         self.register_name(id, SymbolUsage::AnnotationAssigned, location)?;

From d0c436949ec51f38fb7fd87396b29b098bd587f3 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Thu, 30 Sep 2021 03:59:18 +0900
Subject: [PATCH 210/459] Derive Default for CompileOpts

---
 src/compile.rs | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 0ee81d35..99cd9456 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -64,19 +64,13 @@ struct Compiler {
     opts: CompileOpts,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Default)]
 pub struct CompileOpts {
     /// How optimized the bytecode output should be; any optimize > 0 does
     /// not emit assert statements
     pub optimize: u8,
 }
 
-impl Default for CompileOpts {
-    fn default() -> Self {
-        CompileOpts { optimize: 0 }
-    }
-}
-
 #[derive(Debug, Clone, Copy)]
 struct CompileContext {
     loop_data: Option<(ir::BlockIdx, ir::BlockIdx)>,

From be92bc5243c325a5869942ead5127dcecac422d3 Mon Sep 17 00:00:00 2001
From: Alex Muscar <alex@muscar.eu>
Date: Sun, 3 Oct 2021 11:46:21 +0100
Subject: [PATCH 211/459] fix: forbid stores to the __debug__ builtin #3203

According to [the docs][1], `__debug__` is treated as a constant by the
interpreter.

This patch adds some checks to bring RustPython's behaviour in line with
cpython's.

[1]: https://docs.python.org/3/library/constants.html#built-in-constants
---
 src/compile.rs | 88 ++++++++++++++++++++++++++++++++++----------------
 1 file changed, 60 insertions(+), 28 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 99cd9456..5aa1ab55 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -22,6 +22,7 @@ use std::borrow::Cow;
 
 type CompileResult<T> = Result<T, CompileError>;
 
+#[derive(PartialEq, Eq, Clone, Copy)]
 enum NameUsage {
     Load,
     Store,
@@ -51,6 +52,13 @@ impl CallType {
     }
 }
 
+fn is_forbidden_name(name: &str) -> bool {
+    // See https://docs.python.org/3/library/constants.html#built-in-constants
+    const BUILTIN_CONSTANTS: &[&str] = &["__debug__"];
+
+    BUILTIN_CONSTANTS.contains(&name)
+}
+
 /// Main structure holding the state of compilation.
 struct Compiler {
     code_stack: Vec<CodeInfo>,
@@ -375,26 +383,39 @@ impl Compiler {
         Ok(())
     }
 
-    fn load_name(&mut self, name: &str) {
+    fn load_name(&mut self, name: &str) -> CompileResult<()> {
         self.compile_name(name, NameUsage::Load)
     }
 
     fn store_name(&mut self, name: &str) -> CompileResult<()> {
-        if name.eq("__debug__") {
-            return Err(self.error(CompileErrorType::SyntaxError(
-                "cannot assign to __debug__".to_owned(),
-            )));
-        }
-        self.compile_name(name, NameUsage::Store);
-        Ok(())
+        self.compile_name(name, NameUsage::Store)
     }
 
     fn mangle<'a>(&self, name: &'a str) -> Cow<'a, str> {
         symboltable::mangle_name(self.class_name.as_deref(), name)
     }
 
-    fn compile_name(&mut self, name: &str, usage: NameUsage) {
+    fn check_forbidden_name(&self, name: &str, usage: NameUsage) -> CompileResult<()> {
+        if usage == NameUsage::Store && is_forbidden_name(name) {
+            return Err(self.error(CompileErrorType::SyntaxError(format!(
+                "cannot assign to {}",
+                name
+            ))));
+        }
+        if usage == NameUsage::Delete && is_forbidden_name(name) {
+            return Err(self.error(CompileErrorType::SyntaxError(format!(
+                "cannot delete {}",
+                name
+            ))));
+        }
+        Ok(())
+    }
+
+    fn compile_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> {
         let name = self.mangle(name);
+
+        self.check_forbidden_name(&name, usage)?;
+
         let symbol_table = self.symbol_table_stack.last().unwrap();
         let symbol = symbol_table.lookup(name.as_ref()).expect(
             "The symbol must be present in the symbol table, even when it is undefined in python.",
@@ -461,6 +482,8 @@ impl Compiler {
             },
         };
         self.emit(op(idx as u32));
+
+        Ok(())
     }
 
     fn compile_statement(&mut self, statement: &ast::Stmt) -> CompileResult<()> {
@@ -493,9 +516,9 @@ impl Compiler {
                             let idx = self.name(part);
                             self.emit(Instruction::LoadAttr { idx });
                         }
-                        self.store_name(alias)?;
+                        self.store_name(alias)?
                     } else {
-                        self.store_name(name.name.split('.').next().unwrap())?;
+                        self.store_name(name.name.split('.').next().unwrap())?
                     }
                 }
             }
@@ -551,9 +574,9 @@ impl Compiler {
 
                         // Store module under proper name:
                         if let Some(alias) = &name.asname {
-                            self.store_name(alias)?;
+                            self.store_name(alias)?
                         } else {
-                            self.store_name(&name.name)?;
+                            self.store_name(&name.name)?
                         }
                     }
 
@@ -772,10 +795,9 @@ impl Compiler {
 
     fn compile_delete(&mut self, expression: &ast::Expr) -> CompileResult<()> {
         match &expression.node {
-            ast::ExprKind::Name { id, .. } => {
-                self.compile_name(id, NameUsage::Delete);
-            }
+            ast::ExprKind::Name { id, .. } => self.compile_name(id, NameUsage::Delete)?,
             ast::ExprKind::Attribute { value, attr, .. } => {
+                self.check_forbidden_name(attr, NameUsage::Delete)?;
                 self.compile_expression(value)?;
                 let idx = self.name(attr);
                 self.emit(Instruction::DeleteAttr { idx });
@@ -852,6 +874,12 @@ impl Compiler {
             .chain(&args.args)
             .chain(&args.kwonlyargs);
         for name in args_iter {
+            if Compiler::is_forbidden_arg_name(&name.node.arg) {
+                return Err(self.error(CompileErrorType::SyntaxError(format!(
+                    "cannot assign to {}",
+                    &name.node.arg
+                ))));
+            }
             self.varname(&name.node.arg);
         }
 
@@ -935,7 +963,7 @@ impl Compiler {
 
                 // We have a match, store in name (except x as y)
                 if let Some(alias) = name {
-                    self.store_name(alias)?;
+                    self.store_name(alias)?
                 } else {
                     // Drop exception from top of stack:
                     self.emit(Instruction::Pop);
@@ -992,6 +1020,10 @@ impl Compiler {
         Ok(())
     }
 
+    fn is_forbidden_arg_name(name: &str) -> bool {
+        is_forbidden_name(name)
+    }
+
     fn compile_function_def(
         &mut self,
         name: &str,
@@ -1107,9 +1139,7 @@ impl Compiler {
 
         self.apply_decorators(decorator_list);
 
-        self.store_name(name)?;
-
-        Ok(())
+        self.store_name(name)
     }
 
     fn build_closure(&mut self, code: &CodeObject) -> bool {
@@ -1268,8 +1298,7 @@ impl Compiler {
 
         self.apply_decorators(decorator_list);
 
-        self.store_name(name)?;
-        Ok(())
+        self.store_name(name)
     }
 
     fn load_docstring(&mut self, doc_str: Option<String>) {
@@ -1553,15 +1582,14 @@ impl Compiler {
 
     fn compile_store(&mut self, target: &ast::Expr) -> CompileResult<()> {
         match &target.node {
-            ast::ExprKind::Name { id, .. } => {
-                self.store_name(id)?;
-            }
+            ast::ExprKind::Name { id, .. } => self.store_name(id)?,
             ast::ExprKind::Subscript { value, slice, .. } => {
                 self.compile_expression(value)?;
                 self.compile_expression(slice)?;
                 self.emit(Instruction::StoreSubscript);
             }
             ast::ExprKind::Attribute { value, attr, .. } => {
+                self.check_forbidden_name(attr, NameUsage::Store)?;
                 self.compile_expression(value)?;
                 let idx = self.name(attr);
                 self.emit(Instruction::StoreAttr { idx });
@@ -1950,9 +1978,7 @@ impl Compiler {
                     conversion: compile_conversion_flag(*conversion),
                 });
             }
-            Name { id, .. } => {
-                self.load_name(id);
-            }
+            Name { id, .. } => self.load_name(id)?,
             Lambda { args, body } => {
                 let prev_ctx = self.ctx;
                 self.ctx = CompileContext {
@@ -2153,6 +2179,12 @@ impl Compiler {
         let (size, unpack) = self.gather_elements(additional_positional, args)?;
         let has_double_star = keywords.iter().any(|k| k.node.arg.is_none());
 
+        for keyword in keywords {
+            if let Some(name) = &keyword.node.arg {
+                self.check_forbidden_name(name, NameUsage::Store)?;
+            }
+        }
+
         let call = if unpack || has_double_star {
             // Create a tuple with positional args:
             self.emit(Instruction::BuildTuple { size, unpack });

From e038799297e60dfcedfa602aa414bb23758c5f64 Mon Sep 17 00:00:00 2001
From: Alex Muscar <src@muscar.eu>
Date: Mon, 4 Oct 2021 17:19:41 +0100
Subject: [PATCH 212/459] Consolidate error handling for builtin "constants"
 mutations

Co-authored-by: Jeong YunWon <youknowone@users.noreply.github.com>
---
 src/compile.rs | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 5aa1ab55..09897d85 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -396,19 +396,12 @@ impl Compiler {
     }
 
     fn check_forbidden_name(&self, name: &str, usage: NameUsage) -> CompileResult<()> {
-        if usage == NameUsage::Store && is_forbidden_name(name) {
-            return Err(self.error(CompileErrorType::SyntaxError(format!(
-                "cannot assign to {}",
-                name
-            ))));
-        }
-        if usage == NameUsage::Delete && is_forbidden_name(name) {
-            return Err(self.error(CompileErrorType::SyntaxError(format!(
-                "cannot delete {}",
-                name
-            ))));
-        }
-        Ok(())
+        let msg = match usage {
+            NameUsage::Store if is_forbidden_name(name) => "cannot assign to",
+            NameUsage::Delete if is_forbidden_name(name) => "cannot delete",
+            _ => return Ok(()),
+        };
+        Err(self.error(CompileErrorType::SyntaxError(format!("{} {}", msg, name))))
     }
 
     fn compile_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> {

From 6a19201eb4b5488613c567ab893d48cc20f753a6 Mon Sep 17 00:00:00 2001
From: jfh <jfh@imijmi.eu>
Date: Thu, 7 Oct 2021 17:37:24 +0300
Subject: [PATCH 213/459] Fix warnings for doc build.

---
 src/compile.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 09897d85..91781a25 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -2,8 +2,8 @@
 //! Take an AST and transform it into bytecode
 //!
 //! Inspirational code:
-//!   https://github.com/python/cpython/blob/main/Python/compile.c
-//!   https://github.com/micropython/micropython/blob/master/py/compile.c
+//!   <https://github.com/python/cpython/blob/main/Python/compile.c>
+//!   <https://github.com/micropython/micropython/blob/master/py/compile.c>
 
 use crate::ir::{self, CodeInfo};
 pub use crate::mode::Mode;

From 9d98f95a53fe577a62b734cd168a1dbef4f4e93a Mon Sep 17 00:00:00 2001
From: Nixon Enraght-Moony <nixon.emoony@gmail.com>
Date: Fri, 8 Oct 2021 23:01:28 +0100
Subject: [PATCH 214/459] Bless Tests

This was broken by 54ab9cb54a0a9a637e1013a405dccdd8c52442a1

Closes #3253
---
 ...ompile__tests__nested_double_async_with.snap | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap b/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
index b6f7d64f..a942f2e8 100644
--- a/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
+++ b/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
@@ -1,8 +1,9 @@
 ---
 source: compiler/src/compile.rs
 expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n    with self.subTest(type=type(stop_exc)):\n        try:\n            async with woohoo():\n                raise stop_exc\n        except Exception as ex:\n            self.assertIs(ex, stop_exc)\n        else:\n            self.fail(f'{stop_exc} was suppressed')\n\")"
+
 ---
-                 0 SetupLoop
+                 0 SetupLoop            (69)
                  1 LoadNameAny          (0, StopIteration)
                  2 LoadConst            ("spam")
                  3 CallFunctionPositional (1)
@@ -14,12 +15,12 @@ expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyn
           >>     9 ForIter              (68)
                 10 StoreLocal           (2, stop_exc)
                 11 LoadNameAny          (3, self)
-                12 LoadAttr             (subTest)
+                12 LoadMethod           (subTest)
                 13 LoadNameAny          (5, type)
                 14 LoadNameAny          (2, stop_exc)
                 15 CallFunctionPositional (1)
                 16 LoadConst            (("type"))
-                17 CallFunctionKeyword  (1)
+                17 CallMethodKeyword    (1)
                 18 SetupWith            (65)
                 19 Pop
                 20 SetupExcept          (40)
@@ -48,22 +49,22 @@ expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyn
                 43 JumpIfFalse          (53)
                 44 StoreLocal           (8, ex)
                 45 LoadNameAny          (3, self)
-                46 LoadAttr             (assertIs)
+                46 LoadMethod           (assertIs)
                 47 LoadNameAny          (8, ex)
                 48 LoadNameAny          (2, stop_exc)
-                49 CallFunctionPositional (2)
+                49 CallMethodPositional (2)
                 50 Pop
                 51 PopException
                 52 Jump                 (63)
           >>    53 Raise                (Reraise)
           >>    54 LoadNameAny          (3, self)
-                55 LoadAttr             (fail)
+                55 LoadMethod           (fail)
                 56 LoadConst            ("")
                 57 LoadNameAny          (2, stop_exc)
                 58 FormatValue          (None)
                 59 LoadConst            (" was suppressed")
                 60 BuildString          (2)
-                61 CallFunctionPositional (1)
+                61 CallMethodPositional (1)
                 62 Pop
           >>    63 PopBlock
                 64 EnterFinally
@@ -71,6 +72,6 @@ expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyn
                 66 WithCleanupFinish
                 67 Jump                 (9)
           >>    68 PopBlock
-                69 LoadConst            (None)
+          >>    69 LoadConst            (None)
                 70 ReturnValue
 

From 42af29c79c8e88ad02dc33c961c20c3ef69f8e86 Mon Sep 17 00:00:00 2001
From: Noa <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 30 Aug 2021 00:36:40 -0500
Subject: [PATCH 215/459] Add unparse feature to ast

---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 891b26df..94d587c6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,7 +11,7 @@ edition = "2018"
 indexmap = "1.0"
 itertools = "0.10.0"
 rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
-rustpython-ast = { path = "../ast" }
+rustpython-ast = { path = "../ast", features = ["unparse"] }
 num-complex = { version = "0.4.0", features = ["serde"] }
 num-traits = "0.2"
 log = "0.4"

From 4c3c5c5ed1f0c1ad1fa68a293c344e903887de4b Mon Sep 17 00:00:00 2001
From: Noa <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 30 Aug 2021 00:50:44 -0500
Subject: [PATCH 216/459] Add the __future__.annotations feature

---
 src/compile.rs     | 21 +++++++++++++++++----
 src/symboltable.rs | 25 ++++++++++++++++++++++---
 2 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 91781a25..2d877bfb 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -67,6 +67,7 @@ struct Compiler {
     current_source_location: ast::Location,
     qualified_path: Vec<String>,
     done_with_future_stmts: bool,
+    future_annotations: bool,
     ctx: CompileContext,
     class_name: Option<String>,
     opts: CompileOpts,
@@ -196,6 +197,7 @@ impl Compiler {
             current_source_location: ast::Location::default(),
             qualified_path: Vec::new(),
             done_with_future_stmts: false,
+            future_annotations: false,
             ctx: CompileContext {
                 loop_data: None,
                 in_class: false,
@@ -1081,7 +1083,7 @@ impl Compiler {
                 value: "return".to_owned(),
             });
             // value:
-            self.compile_expression(annotation)?;
+            self.compile_annotation(annotation)?;
             num_annotations += 1;
         }
 
@@ -1096,7 +1098,7 @@ impl Compiler {
                 self.emit_constant(ConstantData::Str {
                     value: self.mangle(&arg.node.arg).into_owned(),
                 });
-                self.compile_expression(annotation)?;
+                self.compile_annotation(annotation)?;
                 num_annotations += 1;
             }
         }
@@ -1538,6 +1540,17 @@ impl Compiler {
         Ok(())
     }
 
+    fn compile_annotation(&mut self, annotation: &ast::Expr) -> CompileResult<()> {
+        if self.future_annotations {
+            self.emit_constant(ConstantData::Str {
+                value: annotation.to_string(),
+            });
+        } else {
+            self.compile_expression(annotation)?;
+        }
+        Ok(())
+    }
+
     fn compile_annotated_assign(
         &mut self,
         target: &ast::Expr,
@@ -1555,7 +1568,7 @@ impl Compiler {
         }
 
         // Compile annotation:
-        self.compile_expression(annotation)?;
+        self.compile_annotation(annotation)?;
 
         if let ast::ExprKind::Name { id, .. } = &target.node {
             // Store as dict entry in __annotations__ dict:
@@ -2423,7 +2436,7 @@ impl Compiler {
                 "nested_scopes" | "generators" | "division" | "absolute_import"
                 | "with_statement" | "print_function" | "unicode_literals" => {}
                 // "generator_stop" => {}
-                // "annotations" => {}
+                "annotations" => self.future_annotations = true,
                 other => {
                     return Err(self.error(CompileErrorType::InvalidFutureFeature(other.to_owned())))
                 }
diff --git a/src/symboltable.rs b/src/symboltable.rs
index c0fd9a22..602861f9 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -537,6 +537,7 @@ struct SymbolTableBuilder {
     class_name: Option<String>,
     // Scope stack.
     tables: Vec<SymbolTable>,
+    future_annotations: bool,
 }
 
 /// Enum to indicate in what mode an expression
@@ -557,6 +558,7 @@ impl SymbolTableBuilder {
         let mut this = Self {
             class_name: None,
             tables: vec![],
+            future_annotations: false,
         };
         this.enter_scope("top", SymbolTableType::Module, 0);
         this
@@ -617,14 +619,31 @@ impl SymbolTableBuilder {
 
     fn scan_parameter_annotation(&mut self, parameter: &ast::Arg) -> SymbolTableResult {
         if let Some(annotation) = &parameter.node.annotation {
-            self.scan_expression(annotation, ExpressionContext::Load)?;
+            self.scan_annotation(annotation)?;
         }
         Ok(())
     }
 
+    fn scan_annotation(&mut self, annotation: &ast::Expr) -> SymbolTableResult {
+        if self.future_annotations {
+            Ok(())
+        } else {
+            self.scan_expression(annotation, ExpressionContext::Load)
+        }
+    }
+
     fn scan_statement(&mut self, statement: &ast::Stmt) -> SymbolTableResult {
         use ast::StmtKind::*;
         let location = statement.location;
+        if let ImportFrom { module, names, .. } = &statement.node {
+            if module.as_deref() == Some("__future__") {
+                for feature in names {
+                    if feature.name == "annotations" {
+                        self.future_annotations = true;
+                    }
+                }
+            }
+        }
         match &statement.node {
             Global { names } => {
                 for name in names {
@@ -655,7 +674,7 @@ impl SymbolTableBuilder {
                 self.scan_expressions(decorator_list, ExpressionContext::Load)?;
                 self.register_name(name, SymbolUsage::Assigned, location)?;
                 if let Some(expression) = returns {
-                    self.scan_expression(expression, ExpressionContext::Load)?;
+                    self.scan_annotation(expression)?;
                 }
                 self.enter_function(name, args, location.row())?;
                 self.scan_statements(body)?;
@@ -769,7 +788,7 @@ impl SymbolTableBuilder {
                         self.scan_expression(target, ExpressionContext::Store)?;
                     }
                 }
-                self.scan_expression(annotation, ExpressionContext::Load)?;
+                self.scan_annotation(annotation)?;
                 if let Some(value) = value {
                     self.scan_expression(value, ExpressionContext::Load)?;
                 }

From 56b0a7a805d76046da5f05e2b4059c9847222ea4 Mon Sep 17 00:00:00 2001
From: Noa <33094578+coolreader18@users.noreply.github.com>
Date: Thu, 21 Oct 2021 11:46:24 -0500
Subject: [PATCH 217/459] Migrate to 2021 edition

---
 Cargo.toml           | 2 +-
 porcelain/Cargo.toml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 94d587c6..bd3960a1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -5,7 +5,7 @@ description = "Compiler for python code into bytecode for the rustpython VM."
 authors = ["RustPython Team"]
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 indexmap = "1.0"
diff --git a/porcelain/Cargo.toml b/porcelain/Cargo.toml
index e07c746c..56bf737e 100644
--- a/porcelain/Cargo.toml
+++ b/porcelain/Cargo.toml
@@ -3,7 +3,7 @@ name = "rustpython-compiler"
 version = "0.1.2"
 description = "A usability wrapper around rustpython-parser and rustpython-compiler-core"
 authors = ["RustPython Team"]
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 thiserror = "1.0"

From c0c3eaad8f690223ddb36488471d7e1a45b13d10 Mon Sep 17 00:00:00 2001
From: Noa <33094578+coolreader18@users.noreply.github.com>
Date: Mon, 13 Sep 2021 13:01:55 -0500
Subject: [PATCH 218/459] Run cargo upgrade

---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index bd3960a1..3a133ab3 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,7 +8,7 @@ license = "MIT"
 edition = "2021"
 
 [dependencies]
-indexmap = "1.0"
+indexmap = "1.7.0"
 itertools = "0.10.0"
 rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
 rustpython-ast = { path = "../ast", features = ["unparse"] }

From d9653b4895f5c098af3726aa33f579f95c89031c Mon Sep 17 00:00:00 2001
From: Noa <coolreader18@gmail.com>
Date: Thu, 13 Jan 2022 23:08:56 -0600
Subject: [PATCH 219/459] Fix clippy warnings with 1.58 & convert some format
 strings to capture-args

---
 src/compile.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/compile.rs b/src/compile.rs
index 2d877bfb..7d7d99ef 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -88,6 +88,7 @@ struct CompileContext {
 }
 
 #[derive(Debug, Clone, Copy, PartialEq)]
+#[allow(clippy::enum_variant_names)]
 enum FunctionContext {
     NoFunction,
     Function,

From e5eb2739b700992f8169518b32d54414c43bddad Mon Sep 17 00:00:00 2001
From: Noa <coolreader18@gmail.com>
Date: Thu, 10 Mar 2022 13:27:38 -0600
Subject: [PATCH 220/459] Fix augassign double-loading the container of the
 target

---
 src/compile.rs | 69 ++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 61 insertions(+), 8 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 7d7d99ef..fafd8d57 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -763,14 +763,7 @@ impl Compiler {
                     self.compile_store(target)?;
                 }
             }
-            AugAssign { target, op, value } => {
-                self.compile_expression(target)?;
-                self.compile_expression(value)?;
-
-                // Perform operation:
-                self.compile_op(op, true);
-                self.compile_store(target)?;
-            }
+            AugAssign { target, op, value } => self.compile_augassign(target, op, value)?,
             AnnAssign {
                 target,
                 annotation,
@@ -1652,6 +1645,66 @@ impl Compiler {
         Ok(())
     }
 
+    fn compile_augassign(
+        &mut self,
+        target: &ast::Expr,
+        op: &ast::Operator,
+        value: &ast::Expr,
+    ) -> CompileResult<()> {
+        enum AugAssignKind<'a> {
+            Name { id: &'a str },
+            Subscript,
+            Attr { idx: bytecode::NameIdx },
+        }
+
+        let kind = match &target.node {
+            ast::ExprKind::Name { id, .. } => {
+                self.compile_name(id, NameUsage::Load)?;
+                AugAssignKind::Name { id }
+            }
+            ast::ExprKind::Subscript { value, slice, .. } => {
+                self.compile_expression(value)?;
+                self.compile_expression(slice)?;
+                self.emit(Instruction::Duplicate2);
+                self.emit(Instruction::Subscript);
+                AugAssignKind::Subscript
+            }
+            ast::ExprKind::Attribute { value, attr, .. } => {
+                self.check_forbidden_name(attr, NameUsage::Store)?;
+                self.compile_expression(value)?;
+                self.emit(Instruction::Duplicate);
+                let idx = self.name(attr);
+                self.emit(Instruction::LoadAttr { idx });
+                AugAssignKind::Attr { idx }
+            }
+            _ => {
+                return Err(self.error(CompileErrorType::Assign(target.node.name())));
+            }
+        };
+
+        self.compile_expression(value)?;
+        self.compile_op(op, true);
+
+        match kind {
+            AugAssignKind::Name { id } => {
+                // stack: RESULT
+                self.compile_name(id, NameUsage::Store)?;
+            }
+            AugAssignKind::Subscript => {
+                // stack: CONTAINER SLICE RESULT
+                self.emit(Instruction::Rotate { amount: 3 });
+                self.emit(Instruction::StoreSubscript);
+            }
+            AugAssignKind::Attr { idx } => {
+                // stack: CONTAINER RESULT
+                self.emit(Instruction::Rotate { amount: 2 });
+                self.emit(Instruction::StoreAttr { idx });
+            }
+        }
+
+        Ok(())
+    }
+
     fn compile_op(&mut self, op: &ast::Operator, inplace: bool) {
         let op = match op {
             ast::Operator::Add => bytecode::BinaryOperator::Add,

From a4f34f64e98687ed35ff5e100dda50de507faf0e Mon Sep 17 00:00:00 2001
From: Noa <coolreader18@gmail.com>
Date: Thu, 10 Mar 2022 13:38:08 -0600
Subject: [PATCH 221/459] Split Rotate into Rotate2 & Rotate3

---
 src/compile.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index fafd8d57..56dd7d69 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1122,7 +1122,7 @@ impl Compiler {
 
         self.emit(Instruction::Duplicate);
         self.load_docstring(doc_str);
-        self.emit(Instruction::Rotate { amount: 2 });
+        self.emit(Instruction::Rotate2);
         let doc = self.name("__doc__");
         self.emit(Instruction::StoreAttr { idx: doc });
 
@@ -1498,7 +1498,7 @@ impl Compiler {
             self.compile_expression(val)?;
             // store rhs for the next comparison in chain
             self.emit(Instruction::Duplicate);
-            self.emit(Instruction::Rotate { amount: 3 });
+            self.emit(Instruction::Rotate3);
 
             self.emit(Instruction::CompareOperation {
                 op: compile_cmpop(op),
@@ -1525,7 +1525,7 @@ impl Compiler {
 
             // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs.
             self.switch_to_block(break_block);
-            self.emit(Instruction::Rotate { amount: 2 });
+            self.emit(Instruction::Rotate2);
             self.emit(Instruction::Pop);
 
             self.switch_to_block(after_block);
@@ -1692,12 +1692,12 @@ impl Compiler {
             }
             AugAssignKind::Subscript => {
                 // stack: CONTAINER SLICE RESULT
-                self.emit(Instruction::Rotate { amount: 3 });
+                self.emit(Instruction::Rotate3);
                 self.emit(Instruction::StoreSubscript);
             }
             AugAssignKind::Attr { idx } => {
                 // stack: CONTAINER RESULT
-                self.emit(Instruction::Rotate { amount: 2 });
+                self.emit(Instruction::Rotate2);
                 self.emit(Instruction::StoreAttr { idx });
             }
         }

From bedb5086676b99f13beb3e1542bf490a4190552b Mon Sep 17 00:00:00 2001
From: Jeong Yunwon <jeong@youknowone.org>
Date: Fri, 15 Apr 2022 23:15:01 +0900
Subject: [PATCH 222/459] clean up imports and useless allow attributes

---
 src/compile.rs     | 1 -
 src/error.rs       | 3 +--
 src/symboltable.rs | 9 +++++----
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 56dd7d69..658e12b9 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -88,7 +88,6 @@ struct CompileContext {
 }
 
 #[derive(Debug, Clone, Copy, PartialEq)]
-#[allow(clippy::enum_variant_names)]
 enum FunctionContext {
     NoFunction,
     Function,
diff --git a/src/error.rs b/src/error.rs
index 6d4b3d98..58b8c6dc 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -1,7 +1,6 @@
 use rustpython_ast::Location;
 
-use std::error::Error;
-use std::fmt;
+use std::{error::Error, fmt};
 
 #[derive(Debug)]
 pub struct CompileError {
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 602861f9..74305ea5 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -7,11 +7,12 @@ load and store instructions for names.
 Inspirational file: https://github.com/python/cpython/blob/main/Python/symtable.c
 */
 
-use crate::error::{CompileError, CompileErrorType};
-use crate::IndexMap;
+use crate::{
+    error::{CompileError, CompileErrorType},
+    IndexMap,
+};
 use rustpython_ast::{self as ast, Location};
-use std::borrow::Cow;
-use std::fmt;
+use std::{borrow::Cow, fmt};
 
 pub fn make_symbol_table(program: &[ast::Stmt]) -> SymbolTableResult<SymbolTable> {
     let mut builder = SymbolTableBuilder::new();

From b53f647e5706ae44a115ba57f48eed9b2eaafb55 Mon Sep 17 00:00:00 2001
From: Jeong Yunwon <jeong@youknowone.org>
Date: Sun, 17 Apr 2022 23:49:39 +0900
Subject: [PATCH 223/459] Upgrade libraries

---
 Cargo.toml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 3a133ab3..d2f9a5f0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,15 +8,15 @@ license = "MIT"
 edition = "2021"
 
 [dependencies]
-indexmap = "1.7.0"
-itertools = "0.10.0"
+indexmap = "1.8.1"
+itertools = "0.10.3"
 rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
 rustpython-ast = { path = "../ast", features = ["unparse"] }
 num-complex = { version = "0.4.0", features = ["serde"] }
-num-traits = "0.2"
-log = "0.4"
-ahash = "0.7.2"
+num-traits = "0.2.14"
+log = "0.4.16"
+ahash = "0.7.6"
 
 [dev-dependencies]
 rustpython-parser = { path = "../parser" }
-insta = "1.7.0"
+insta = "1.14.0"

From 753c0337bc3f8ed870721621e01ed6dfddf7a759 Mon Sep 17 00:00:00 2001
From: Jeong Yunwon <jeong@youknowone.org>
Date: Mon, 2 May 2022 01:51:30 +0900
Subject: [PATCH 224/459] compile_single checks last expr out of loop

---
 src/compile.rs | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 658e12b9..52852afd 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -337,31 +337,30 @@ impl Compiler {
     ) -> CompileResult<()> {
         self.symbol_table_stack.push(symbol_table);
 
-        let mut emitted_return = false;
-
-        for (i, statement) in body.iter().enumerate() {
-            let is_last = i == body.len() - 1;
+        let (last, body) = if let Some(splited) = body.split_last() {
+            splited
+        } else {
+            return Ok(());
+        };
 
+        for statement in body {
             if let ast::StmtKind::Expr { value } = &statement.node {
                 self.compile_expression(value)?;
-
-                if is_last {
-                    self.emit(Instruction::Duplicate);
-                    self.emit(Instruction::PrintExpr);
-                    self.emit(Instruction::ReturnValue);
-                    emitted_return = true;
-                } else {
-                    self.emit(Instruction::PrintExpr);
-                }
+                self.emit(Instruction::PrintExpr);
             } else {
                 self.compile_statement(statement)?;
             }
         }
 
-        if !emitted_return {
+        if let ast::StmtKind::Expr { value } = &last.node {
+            self.compile_expression(value)?;
+            self.emit(Instruction::Duplicate);
+            self.emit(Instruction::PrintExpr);
+        } else {
+            self.compile_statement(last)?;
             self.emit_constant(ConstantData::None);
-            self.emit(Instruction::ReturnValue);
         }
+        self.emit(Instruction::ReturnValue);
 
         Ok(())
     }

From 5e94f56ae49c1690d833feda0c026111f15786f7 Mon Sep 17 00:00:00 2001
From: Jeong Yunwon <jeong@youknowone.org>
Date: Mon, 2 May 2022 02:17:33 +0900
Subject: [PATCH 225/459] compile_impl! => compile_impl

---
 src/compile.rs | 44 ++++++++++++++++++++++++++++++--------------
 1 file changed, 30 insertions(+), 14 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 52852afd..ce580002 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -127,16 +127,20 @@ pub fn compile_top(
     }
 }
 
-macro_rules! compile_impl {
-    ($ast:expr, $source_path:expr, $opts:expr, $st:ident, $compile:ident) => {{
-        let symbol_table = match $st($ast) {
-            Ok(x) => x,
-            Err(e) => return Err(e.into_compile_error($source_path)),
-        };
-        with_compiler($source_path, $opts, |compiler| {
-            compiler.$compile($ast, symbol_table)
-        })
-    }};
+fn compile_impl<Ast: ?Sized>(
+    ast: &Ast,
+    source_path: String,
+    opts: CompileOpts,
+    make_symbol_table: fn(&Ast) -> Result<SymbolTable, symboltable::SymbolTableError>,
+    compile: fn(&mut Compiler, &Ast, SymbolTable) -> CompileResult<()>,
+) -> CompileResult<CodeObject> {
+    let symbol_table = match make_symbol_table(ast) {
+        Ok(x) => x,
+        Err(e) => return Err(e.into_compile_error(source_path)),
+    };
+    with_compiler(source_path, opts, |compiler| {
+        compile(compiler, ast, symbol_table)
+    })
 }
 
 /// Compile a standard Python program to bytecode
@@ -145,7 +149,13 @@ pub fn compile_program(
     source_path: String,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    compile_impl!(ast, source_path, opts, make_symbol_table, compile_program)
+    compile_impl(
+        ast,
+        source_path,
+        opts,
+        make_symbol_table,
+        Compiler::compile_program,
+    )
 }
 
 /// Compile a Python program to bytecode for the context of a REPL
@@ -154,12 +164,12 @@ pub fn compile_program_single(
     source_path: String,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    compile_impl!(
+    compile_impl(
         ast,
         source_path,
         opts,
         make_symbol_table,
-        compile_program_single
+        Compiler::compile_program_single,
     )
 }
 
@@ -168,7 +178,13 @@ pub fn compile_expression(
     source_path: String,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    compile_impl!(ast, source_path, opts, make_symbol_table_expr, compile_eval)
+    compile_impl(
+        ast,
+        source_path,
+        opts,
+        make_symbol_table_expr,
+        Compiler::compile_eval,
+    )
 }
 
 impl Compiler {

From f97d1ed5d96930ac694a1f834f6c1e798509deb7 Mon Sep 17 00:00:00 2001
From: Jeong Yunwon <jeong@youknowone.org>
Date: Mon, 2 May 2022 02:35:40 +0900
Subject: [PATCH 226/459] remove with_compiler

---
 src/compile.rs | 23 +++++++----------------
 1 file changed, 7 insertions(+), 16 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index ce580002..759081f9 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -100,19 +100,6 @@ impl CompileContext {
     }
 }
 
-/// A helper function for the shared code of the different compile functions
-fn with_compiler(
-    source_path: String,
-    opts: CompileOpts,
-    f: impl FnOnce(&mut Compiler) -> CompileResult<()>,
-) -> CompileResult<CodeObject> {
-    let mut compiler = Compiler::new(opts, source_path, "<module>".to_owned());
-    f(&mut compiler)?;
-    let code = compiler.pop_code_object();
-    trace!("Compilation completed: {:?}", code);
-    Ok(code)
-}
-
 /// Compile an ast::Mod produced from rustpython_parser::parser::parse()
 pub fn compile_top(
     ast: &ast::Mod,
@@ -127,6 +114,7 @@ pub fn compile_top(
     }
 }
 
+/// A helper function for the shared code of the different compile functions
 fn compile_impl<Ast: ?Sized>(
     ast: &Ast,
     source_path: String,
@@ -138,9 +126,12 @@ fn compile_impl<Ast: ?Sized>(
         Ok(x) => x,
         Err(e) => return Err(e.into_compile_error(source_path)),
     };
-    with_compiler(source_path, opts, |compiler| {
-        compile(compiler, ast, symbol_table)
-    })
+
+    let mut compiler = Compiler::new(opts, source_path, "<module>".to_owned());
+    compile(&mut compiler, ast, symbol_table)?;
+    let code = compiler.pop_code_object();
+    trace!("Compilation completed: {:?}", code);
+    Ok(code)
 }
 
 /// Compile a standard Python program to bytecode

From bbf475e212bc64cf24e4680deaf9be317afed3cb Mon Sep 17 00:00:00 2001
From: Jeong Yunwon <jeong@youknowone.org>
Date: Mon, 2 May 2022 02:56:36 +0900
Subject: [PATCH 227/459] compile get_doc -> split_doc to follow split_first
 convention in Rust

---
 src/compile.rs | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 759081f9..4e3f6a43 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -316,7 +316,7 @@ impl Compiler {
         let size_before = self.code_stack.len();
         self.symbol_table_stack.push(symbol_table);
 
-        let (statements, doc) = get_doc(body);
+        let (doc, statements) = split_doc(body);
         if let Some(value) = doc {
             self.emit_constant(ConstantData::Str { value });
             let doc = self.name("__doc__");
@@ -1052,7 +1052,7 @@ impl Compiler {
         let qualified_name = self.qualified_path.join(".");
         self.push_qualified_path("<locals>");
 
-        let (body, doc_str) = get_doc(body);
+        let (doc_str, body) = split_doc(body);
 
         self.compile_statements(body)?;
 
@@ -1171,6 +1171,7 @@ impl Compiler {
         true
     }
 
+    // Python/compile.c find_ann
     fn find_ann(&self, body: &[ast::Stmt]) -> bool {
         use ast::StmtKind::*;
 
@@ -1222,7 +1223,7 @@ impl Compiler {
 
         self.push_output(bytecode::CodeFlags::empty(), 0, 0, 0, name.to_owned());
 
-        let (new_body, doc_str) = get_doc(body);
+        let (doc_str, body) = split_doc(body);
 
         let dunder_name = self.name("__name__");
         self.emit(Instruction::LoadGlobal(dunder_name));
@@ -1240,7 +1241,7 @@ impl Compiler {
         if self.find_ann(body) {
             self.emit(Instruction::SetupAnnotation);
         }
-        self.compile_statements(new_body)?;
+        self.compile_statements(body)?;
 
         let classcell_idx = self
             .code_stack
@@ -2572,15 +2573,15 @@ impl Compiler {
     }
 }
 
-fn get_doc(body: &[ast::Stmt]) -> (&[ast::Stmt], Option<String>) {
+fn split_doc(body: &[ast::Stmt]) -> (Option<String>, &[ast::Stmt]) {
     if let Some((val, body_rest)) = body.split_first() {
         if let ast::StmtKind::Expr { value } = &val.node {
             if let Some(doc) = try_get_constant_string(std::slice::from_ref(value)) {
-                return (body_rest, Some(doc));
+                return (Some(doc), body_rest);
             }
         }
     }
-    (body, None)
+    (None, body)
 }
 
 fn try_get_constant_string(values: &[ast::Expr]) -> Option<String> {

From 33ddf76b269ae573dafaca8e50e5d0e6d2939e8f Mon Sep 17 00:00:00 2001
From: Jeong Yunwon <jeong@youknowone.org>
Date: Mon, 2 May 2022 18:07:58 +0900
Subject: [PATCH 228/459] fn -> impl FnOnce

https://github.com/RustPython/RustPython/pull/3674#discussion_r862511425
---
 src/compile.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 4e3f6a43..7864c03c 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -119,8 +119,8 @@ fn compile_impl<Ast: ?Sized>(
     ast: &Ast,
     source_path: String,
     opts: CompileOpts,
-    make_symbol_table: fn(&Ast) -> Result<SymbolTable, symboltable::SymbolTableError>,
-    compile: fn(&mut Compiler, &Ast, SymbolTable) -> CompileResult<()>,
+    make_symbol_table: impl FnOnce(&Ast) -> Result<SymbolTable, symboltable::SymbolTableError>,
+    compile: impl FnOnce(&mut Compiler, &Ast, SymbolTable) -> CompileResult<()>,
 ) -> CompileResult<CodeObject> {
     let symbol_table = match make_symbol_table(ast) {
         Ok(x) => x,

From f7217814996f93cd54e910b105b2b7ea1b0f3aa3 Mon Sep 17 00:00:00 2001
From: Jeong Yunwon <jeong@youknowone.org>
Date: Thu, 12 May 2022 02:16:00 +0900
Subject: [PATCH 229/459] split comapre and richcompare instruction

---
 src/compile.rs                                | 52 ++++++++++++-------
 ...pile__tests__nested_double_async_with.snap |  2 +-
 2 files changed, 35 insertions(+), 19 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 7864c03c..72c50b80 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -946,8 +946,8 @@ impl Compiler {
 
                 // Check exception type:
                 self.compile_expression(exc_type)?;
-                self.emit(Instruction::CompareOperation {
-                    op: bytecode::ComparisonOperator::ExceptionMatch,
+                self.emit(Instruction::TestOperation {
+                    op: bytecode::TestOperator::ExceptionMatch,
                 });
 
                 // We cannot handle this exception type:
@@ -1468,16 +1468,36 @@ impl Compiler {
         let (last_val, mid_vals) = vals.split_last().unwrap();
 
         let compile_cmpop = |op: &ast::Cmpop| match op {
-            ast::Cmpop::Eq => bytecode::ComparisonOperator::Equal,
-            ast::Cmpop::NotEq => bytecode::ComparisonOperator::NotEqual,
-            ast::Cmpop::Lt => bytecode::ComparisonOperator::Less,
-            ast::Cmpop::LtE => bytecode::ComparisonOperator::LessOrEqual,
-            ast::Cmpop::Gt => bytecode::ComparisonOperator::Greater,
-            ast::Cmpop::GtE => bytecode::ComparisonOperator::GreaterOrEqual,
-            ast::Cmpop::In => bytecode::ComparisonOperator::In,
-            ast::Cmpop::NotIn => bytecode::ComparisonOperator::NotIn,
-            ast::Cmpop::Is => bytecode::ComparisonOperator::Is,
-            ast::Cmpop::IsNot => bytecode::ComparisonOperator::IsNot,
+            ast::Cmpop::Eq => Instruction::CompareOperation {
+                op: bytecode::CompareOperator::Equal,
+            },
+            ast::Cmpop::NotEq => Instruction::CompareOperation {
+                op: bytecode::CompareOperator::NotEqual,
+            },
+            ast::Cmpop::Lt => Instruction::CompareOperation {
+                op: bytecode::CompareOperator::Less,
+            },
+            ast::Cmpop::LtE => Instruction::CompareOperation {
+                op: bytecode::CompareOperator::LessOrEqual,
+            },
+            ast::Cmpop::Gt => Instruction::CompareOperation {
+                op: bytecode::CompareOperator::Greater,
+            },
+            ast::Cmpop::GtE => Instruction::CompareOperation {
+                op: bytecode::CompareOperator::GreaterOrEqual,
+            },
+            ast::Cmpop::In => Instruction::TestOperation {
+                op: bytecode::TestOperator::In,
+            },
+            ast::Cmpop::NotIn => Instruction::TestOperation {
+                op: bytecode::TestOperator::NotIn,
+            },
+            ast::Cmpop::Is => Instruction::TestOperation {
+                op: bytecode::TestOperator::Is,
+            },
+            ast::Cmpop::IsNot => Instruction::TestOperation {
+                op: bytecode::TestOperator::IsNot,
+            },
         };
 
         // a == b == c == d
@@ -1506,9 +1526,7 @@ impl Compiler {
             self.emit(Instruction::Duplicate);
             self.emit(Instruction::Rotate3);
 
-            self.emit(Instruction::CompareOperation {
-                op: compile_cmpop(op),
-            });
+            self.emit(compile_cmpop(op));
 
             // if comparison result is false, we break with this value; if true, try the next one.
             if let Some((break_block, _)) = end_blocks {
@@ -1520,9 +1538,7 @@ impl Compiler {
 
         // handle the last comparison
         self.compile_expression(last_val)?;
-        self.emit(Instruction::CompareOperation {
-            op: compile_cmpop(last_op),
-        });
+        self.emit(compile_cmpop(last_op));
 
         if let Some((break_block, after_block)) = end_blocks {
             self.emit(Instruction::Jump {
diff --git a/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap b/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
index a942f2e8..adc63bdf 100644
--- a/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
+++ b/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
@@ -45,7 +45,7 @@ expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyn
                 39 Jump                 (54)
           >>    40 Duplicate
                 41 LoadNameAny          (7, Exception)
-                42 CompareOperation     (ExceptionMatch)
+                42 TestOperation        (ExceptionMatch)
                 43 JumpIfFalse          (53)
                 44 StoreLocal           (8, ex)
                 45 LoadNameAny          (3, self)

From be43bfaa800d79d9a4f5a71c447142728d092a7e Mon Sep 17 00:00:00 2001
From: Jeong Yunwon <jeong@youknowone.org>
Date: Thu, 12 May 2022 03:59:57 +0900
Subject: [PATCH 230/459] share comparisonop between bytecode and vm

---
 src/compile.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 72c50b80..b5473a0e 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1469,22 +1469,22 @@ impl Compiler {
 
         let compile_cmpop = |op: &ast::Cmpop| match op {
             ast::Cmpop::Eq => Instruction::CompareOperation {
-                op: bytecode::CompareOperator::Equal,
+                op: bytecode::ComparisonOperator::Equal,
             },
             ast::Cmpop::NotEq => Instruction::CompareOperation {
-                op: bytecode::CompareOperator::NotEqual,
+                op: bytecode::ComparisonOperator::NotEqual,
             },
             ast::Cmpop::Lt => Instruction::CompareOperation {
-                op: bytecode::CompareOperator::Less,
+                op: bytecode::ComparisonOperator::Less,
             },
             ast::Cmpop::LtE => Instruction::CompareOperation {
-                op: bytecode::CompareOperator::LessOrEqual,
+                op: bytecode::ComparisonOperator::LessOrEqual,
             },
             ast::Cmpop::Gt => Instruction::CompareOperation {
-                op: bytecode::CompareOperator::Greater,
+                op: bytecode::ComparisonOperator::Greater,
             },
             ast::Cmpop::GtE => Instruction::CompareOperation {
-                op: bytecode::CompareOperator::GreaterOrEqual,
+                op: bytecode::ComparisonOperator::GreaterOrEqual,
             },
             ast::Cmpop::In => Instruction::TestOperation {
                 op: bytecode::TestOperator::In,

From dc1d6616ca52e5409fb854713d34bbf95d376476 Mon Sep 17 00:00:00 2001
From: snowapril <sinjihng@gmail.com>
Date: Sun, 15 May 2022 22:00:25 +0900
Subject: [PATCH 231/459] Add missing codes for empty statement body

Fix #3704
Compared to previous version of `compile_program_single`, below two
codes are skipped when an empty body is given.
```rust
self.emit_constant(ConstantData::None);
self.emit(Instruction::ReturnValue);
```

Signed-off-by: snowapril <sinjihng@gmail.com>
---
 src/compile.rs | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index b5473a0e..00860de5 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -344,31 +344,29 @@ impl Compiler {
     ) -> CompileResult<()> {
         self.symbol_table_stack.push(symbol_table);
 
-        let (last, body) = if let Some(splited) = body.split_last() {
-            splited
-        } else {
-            return Ok(());
-        };
+        if let Some((last, body)) = body.split_last() {
+            for statement in body {
+                if let ast::StmtKind::Expr { value } = &statement.node {
+                    self.compile_expression(value)?;
+                    self.emit(Instruction::PrintExpr);
+                } else {
+                    self.compile_statement(statement)?;
+                }
+            }
 
-        for statement in body {
-            if let ast::StmtKind::Expr { value } = &statement.node {
+            if let ast::StmtKind::Expr { value } = &last.node {
                 self.compile_expression(value)?;
+                self.emit(Instruction::Duplicate);
                 self.emit(Instruction::PrintExpr);
             } else {
-                self.compile_statement(statement)?;
+                self.compile_statement(last)?;
+                self.emit_constant(ConstantData::None);
             }
-        }
-
-        if let ast::StmtKind::Expr { value } = &last.node {
-            self.compile_expression(value)?;
-            self.emit(Instruction::Duplicate);
-            self.emit(Instruction::PrintExpr);
         } else {
-            self.compile_statement(last)?;
             self.emit_constant(ConstantData::None);
-        }
-        self.emit(Instruction::ReturnValue);
+        };
 
+        self.emit(Instruction::ReturnValue);
         Ok(())
     }
 

From da3d41ce27bb7303ae98975bfa2fe26e9c723f88 Mon Sep 17 00:00:00 2001
From: Jeong Yunwon <jeong@youknowone.org>
Date: Fri, 20 May 2022 08:05:12 +0900
Subject: [PATCH 232/459] Fix comprehension instruction evaluation order

---
 src/compile.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 00860de5..4b7017cc 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -2095,7 +2095,7 @@ impl Compiler {
                     &|compiler| {
                         compiler.compile_comprehension_element(elt)?;
                         compiler.emit(Instruction::ListAppend {
-                            i: (1 + generators.len()) as u32,
+                            i: generators.len() as u32,
                         });
                         Ok(())
                     },
@@ -2112,7 +2112,7 @@ impl Compiler {
                     &|compiler| {
                         compiler.compile_comprehension_element(elt)?;
                         compiler.emit(Instruction::SetAdd {
-                            i: (1 + generators.len()) as u32,
+                            i: generators.len() as u32,
                         });
                         Ok(())
                     },
@@ -2137,7 +2137,7 @@ impl Compiler {
                         compiler.compile_expression(value)?;
 
                         compiler.emit(Instruction::MapAddRev {
-                            i: (1 + generators.len()) as u32,
+                            i: generators.len() as u32,
                         });
 
                         Ok(())

From 2262971b12bfdd3239136e9316e5f7752ffbb2b1 Mon Sep 17 00:00:00 2001
From: Jeong Yunwon <jeong@youknowone.org>
Date: Fri, 20 May 2022 08:12:21 +0900
Subject: [PATCH 233/459] Remove Instruction::MapAddRev

---
 src/compile.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index 4b7017cc..3e50e0ec 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -2136,7 +2136,7 @@ impl Compiler {
                         compiler.compile_expression(key)?;
                         compiler.compile_expression(value)?;
 
-                        compiler.emit(Instruction::MapAddRev {
+                        compiler.emit(Instruction::MapAdd {
                             i: generators.len() as u32,
                         });
 

From cb5324d023ed2ee2152035e4605acdb87a4f0d57 Mon Sep 17 00:00:00 2001
From: Jeong Yunwon <jeong@youknowone.org>
Date: Mon, 2 May 2022 04:11:59 +0900
Subject: [PATCH 234/459] Add compile::Mode::BlockExpr

---
 porcelain/src/lib.rs | 11 +++++-----
 src/compile.rs       | 50 +++++++++++++++++++++++++++++++++++++++++++-
 src/mode.rs          |  3 +++
 3 files changed, 58 insertions(+), 6 deletions(-)

diff --git a/porcelain/src/lib.rs b/porcelain/src/lib.rs
index 8308cae0..e593188a 100644
--- a/porcelain/src/lib.rs
+++ b/porcelain/src/lib.rs
@@ -76,12 +76,12 @@ pub fn compile(
     source_path: String,
     opts: CompileOpts,
 ) -> Result<CodeObject, CompileError> {
-    let mode = match mode {
+    let parser_mode = match mode {
         compile::Mode::Exec => parser::Mode::Module,
         compile::Mode::Eval => parser::Mode::Expression,
-        compile::Mode::Single => parser::Mode::Interactive,
+        compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive,
     };
-    let mut ast = match parser::parse(source, mode) {
+    let mut ast = match parser::parse(source, parser_mode) {
         Ok(x) => x,
         Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
     };
@@ -90,7 +90,8 @@ pub fn compile(
             .fold_mod(ast)
             .unwrap_or_else(|e| match e {});
     }
-    compile::compile_top(&ast, source_path, opts).map_err(|e| CompileError::from_compile(e, source))
+    compile::compile_top(&ast, source_path, mode, opts)
+        .map_err(|e| CompileError::from_compile(e, source))
 }
 
 pub fn compile_symtable(
@@ -107,7 +108,7 @@ pub fn compile_symtable(
         };
     }
     let res = match mode {
-        compile::Mode::Exec | compile::Mode::Single => {
+        compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
             let ast = try_parse!(parser::parse_program(source));
             symboltable::make_symbol_table(&ast)
         }
diff --git a/src/compile.rs b/src/compile.rs
index 3e50e0ec..6297c26c 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -104,11 +104,16 @@ impl CompileContext {
 pub fn compile_top(
     ast: &ast::Mod,
     source_path: String,
+    mode: Mode,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
     match ast {
         ast::Mod::Module { body, .. } => compile_program(body, source_path, opts),
-        ast::Mod::Interactive { body } => compile_program_single(body, source_path, opts),
+        ast::Mod::Interactive { body } => match mode {
+            Mode::Single => compile_program_single(body, source_path, opts),
+            Mode::BlockExpr => compile_block_expression(body, source_path, opts),
+            _ => unreachable!("only Single and BlockExpr parsed to Interactive"),
+        },
         ast::Mod::Expression { body } => compile_expression(body, source_path, opts),
         ast::Mod::FunctionType { .. } => panic!("can't compile a FunctionType"),
     }
@@ -164,6 +169,20 @@ pub fn compile_program_single(
     )
 }
 
+pub fn compile_block_expression(
+    ast: &[ast::Stmt],
+    source_path: String,
+    opts: CompileOpts,
+) -> CompileResult<CodeObject> {
+    compile_impl(
+        ast,
+        source_path,
+        opts,
+        make_symbol_table,
+        Compiler::compile_block_expr,
+    )
+}
+
 pub fn compile_expression(
     ast: &ast::Expr,
     source_path: String,
@@ -370,6 +389,35 @@ impl Compiler {
         Ok(())
     }
 
+    fn compile_block_expr(
+        &mut self,
+        body: &[ast::Stmt],
+        symbol_table: SymbolTable,
+    ) -> CompileResult<()> {
+        self.symbol_table_stack.push(symbol_table);
+
+        self.compile_statements(body)?;
+
+        if let Some(last_statement) = body.last() {
+            match last_statement.node {
+                ast::StmtKind::Expr { .. } => {
+                    self.current_block().instructions.pop(); // pop Instruction::Pop
+                }
+                ast::StmtKind::FunctionDef { .. }
+                | ast::StmtKind::AsyncFunctionDef { .. }
+                | ast::StmtKind::ClassDef { .. } => {
+                    let store_inst = self.current_block().instructions.pop().unwrap(); // pop Instruction::Store
+                    self.emit(Instruction::Duplicate);
+                    self.current_block().instructions.push(store_inst);
+                }
+                _ => self.emit_constant(ConstantData::None),
+            }
+        }
+        self.emit(Instruction::ReturnValue);
+
+        Ok(())
+    }
+
     // Compile statement in eval mode:
     fn compile_eval(
         &mut self,
diff --git a/src/mode.rs b/src/mode.rs
index f926309a..b56f226a 100644
--- a/src/mode.rs
+++ b/src/mode.rs
@@ -3,10 +3,13 @@ pub enum Mode {
     Exec,
     Eval,
     Single,
+    BlockExpr,
 }
 
 impl std::str::FromStr for Mode {
     type Err = ModeParseError;
+
+    // To support `builtins.compile()` `mode` argument
     fn from_str(s: &str) -> Result<Self, ModeParseError> {
         match s {
             "exec" => Ok(Mode::Exec),

From 46404a4665945f1a3eb8f0f9af41e11ec1f61eec Mon Sep 17 00:00:00 2001
From: Jeong Yunwon <jeong@youknowone.org>
Date: Fri, 24 Jun 2022 01:13:32 +0900
Subject: [PATCH 235/459] Fix nightly clippy warnings

---
 src/symboltable.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/symboltable.rs b/src/symboltable.rs
index 74305ea5..9a216cdf 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -62,7 +62,7 @@ impl SymbolTable {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum SymbolTableType {
     Module,
     Class,
@@ -83,7 +83,7 @@ impl fmt::Display for SymbolTableType {
 
 /// Indicator for a single symbol what the scope of this symbol is.
 /// The scope can be unknown, which is unfortunate, but not impossible.
-#[derive(Debug, Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum SymbolScope {
     Unknown,
     Local,

From f869bc8371de29b93b654e00f7138eb241a16c08 Mon Sep 17 00:00:00 2001
From: Myeongseon Choi <key262yek@gmail.com>
Date: Wed, 13 Jul 2022 17:31:21 +0900
Subject: [PATCH 236/459] Change error msg "can't" to "cannot"

---
 src/error.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/error.rs b/src/error.rs
index 58b8c6dc..38740aa7 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -40,13 +40,13 @@ pub enum CompileErrorType {
 impl fmt::Display for CompileErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            CompileErrorType::Assign(target) => write!(f, "can't assign to {}", target),
-            CompileErrorType::Delete(target) => write!(f, "can't delete {}", target),
+            CompileErrorType::Assign(target) => write!(f, "cannot assign to {}", target),
+            CompileErrorType::Delete(target) => write!(f, "cannot delete {}", target),
             CompileErrorType::SyntaxError(err) => write!(f, "{}", err.as_str()),
             CompileErrorType::MultipleStarArgs => {
                 write!(f, "two starred expressions in assignment")
             }
-            CompileErrorType::InvalidStarExpr => write!(f, "can't use starred expression here"),
+            CompileErrorType::InvalidStarExpr => write!(f, "cannot use starred expression here"),
             CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"),
             CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"),
             CompileErrorType::InvalidReturn => write!(f, "'return' outside function"),

From 0b331cc916deb815eade9a191f2bfc5f27b28bf1 Mon Sep 17 00:00:00 2001
From: Myeongseon Choi <key262yek@gmail.com>
Date: Thu, 14 Jul 2022 09:22:49 +0900
Subject: [PATCH 237/459] Update del statement `del [x]` available

---
 src/compile.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/compile.rs b/src/compile.rs
index 6297c26c..6f210bb2 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -847,7 +847,7 @@ impl Compiler {
                 self.compile_expression(slice)?;
                 self.emit(Instruction::DeleteSubscript);
             }
-            ast::ExprKind::Tuple { elts, .. } => {
+            ast::ExprKind::Tuple { elts, .. } | ast::ExprKind::List { elts, .. } => {
                 for element in elts {
                     self.compile_delete(element)?;
                 }

From 16d34ad5d2146c2ae96db073131d6203b74c3d27 Mon Sep 17 00:00:00 2001
From: Myeongseon Choi <key262yek@gmail.com>
Date: Wed, 20 Jul 2022 13:25:04 +0900
Subject: [PATCH 238/459] Except operator cases from delete error rule

---
 src/compile.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/compile.rs b/src/compile.rs
index 6f210bb2..93546534 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -852,6 +852,9 @@ impl Compiler {
                     self.compile_delete(element)?;
                 }
             }
+            ast::ExprKind::BinOp { .. } | ast::ExprKind::UnaryOp { .. } => {
+                return Err(self.error(CompileErrorType::Delete("expression")))
+            }
             _ => return Err(self.error(CompileErrorType::Delete(expression.node.name()))),
         }
         Ok(())

From 3bf931fbb02299b8af10bb8623c5531a9fd3fab3 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 10 Aug 2022 01:11:44 +0900
Subject: [PATCH 239/459] Fix comprehension bytecode generation

---
 src/compile.rs | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 93546534..a18635f4 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -2468,11 +2468,6 @@ impl Compiler {
             let loop_block = self.new_block();
             let after_block = self.new_block();
 
-            // Setup for loop:
-            self.emit(Instruction::SetupLoop {
-                break_target: after_block,
-            });
-
             if loop_labels.is_empty() {
                 // Load iterator onto stack (passed as first argument):
                 self.emit(Instruction::LoadFast(arg0));
@@ -2507,7 +2502,6 @@ impl Compiler {
 
             // End of for loop:
             self.switch_to_block(after_block);
-            self.emit(Instruction::PopBlock);
         }
 
         if return_none {

From 6760d213992ae47d03a25bcd76da1d99ef56f8a7 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 10 Aug 2022 06:37:40 +0900
Subject: [PATCH 240/459] Fix break to contain end label

---
 src/compile.rs | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index a18635f4..f183433e 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -761,13 +761,14 @@ impl Compiler {
                     self.switch_to_block(after_block);
                 }
             }
-            Break => {
-                if self.ctx.loop_data.is_some() {
-                    self.emit(Instruction::Break);
-                } else {
+            Break => match self.ctx.loop_data {
+                Some((_, end)) => {
+                    self.emit(Instruction::Break { target: end });
+                }
+                None => {
                     return Err(self.error_loc(CompileErrorType::InvalidBreak, statement.location));
                 }
-            }
+            },
             Continue => match self.ctx.loop_data {
                 Some((start, _)) => {
                     self.emit(Instruction::Continue { target: start });
@@ -1373,8 +1374,7 @@ impl Compiler {
 
         self.compile_jump_if(test, false, else_block)?;
 
-        let was_in_loop = self.ctx.loop_data;
-        self.ctx.loop_data = Some((while_block, after_block));
+        let was_in_loop = self.ctx.loop_data.replace((while_block, after_block));
         self.compile_statements(body)?;
         self.ctx.loop_data = was_in_loop;
         self.emit(Instruction::Jump {
@@ -1487,8 +1487,7 @@ impl Compiler {
             self.compile_store(target)?;
         };
 
-        let was_in_loop = self.ctx.loop_data;
-        self.ctx.loop_data = Some((for_block, after_block));
+        let was_in_loop = self.ctx.loop_data.replace((for_block, after_block));
         self.compile_statements(body)?;
         self.ctx.loop_data = was_in_loop;
         self.emit(Instruction::Jump { target: for_block });

From c3b14002406a1e7fd60e11126c1610c7889efc19 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Thu, 11 Aug 2022 06:14:42 +0900
Subject: [PATCH 241/459] Fix max_stacksize DEBUG to use const

---
 src/ir.rs | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/ir.rs b/src/ir.rs
index 3c7ba9ff..d25b792c 100644
--- a/src/ir.rs
+++ b/src/ir.rs
@@ -167,19 +167,22 @@ impl CodeInfo {
         let mut startdepths = vec![u32::MAX; self.blocks.len()];
         startdepths[0] = 0;
         stack.push(Label(0));
-        let debug = false;
+        const DEBUG: bool = false;
         'process_blocks: while let Some(block) = stack.pop() {
             let mut depth = startdepths[block.0 as usize];
-            if debug {
+            if DEBUG {
                 eprintln!("===BLOCK {}===", block.0);
             }
             let block = &self.blocks[block.0 as usize];
             for i in &block.instructions {
                 let instr = &i.instr;
                 let effect = instr.stack_effect(false);
+                if DEBUG {
+                    eprint!("{instr:?}: {depth} {effect:+} => ");
+                }
                 let new_depth = add_ui(depth, effect);
-                if debug {
-                    eprintln!("{:?}: {:+}, {:+} = {}", instr, effect, depth, new_depth);
+                if DEBUG {
+                    eprintln!("{new_depth}");
                 }
                 if new_depth > maxdepth {
                     maxdepth = new_depth
@@ -205,6 +208,9 @@ impl CodeInfo {
             }
             stackdepth_push(&mut stack, &mut startdepths, block.next, depth);
         }
+        if DEBUG {
+            eprintln!("DONE: {maxdepth}");
+        }
         maxdepth
     }
 }

From 1a09ba0ede3750d23b4e4d06e2af09fdc7fc1ed8 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Thu, 11 Aug 2022 06:22:39 +0900
Subject: [PATCH 242/459] max_stacksize -> max_stackdepth

---
 src/ir.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/ir.rs b/src/ir.rs
index d25b792c..39c3ad1f 100644
--- a/src/ir.rs
+++ b/src/ir.rs
@@ -45,7 +45,7 @@ pub struct CodeInfo {
 }
 impl CodeInfo {
     pub fn finalize_code(mut self, optimize: u8) -> CodeObject {
-        let max_stacksize = self.max_stacksize();
+        let max_stackdepth = self.max_stackdepth();
         let cell2arg = self.cell2arg();
 
         if optimize > 0 {
@@ -101,7 +101,7 @@ impl CodeInfo {
             first_line_number,
             obj_name,
 
-            max_stacksize,
+            max_stackdepth,
             instructions: instructions.into_boxed_slice(),
             locations: locations.into_boxed_slice(),
             constants: constants.into_iter().collect(),
@@ -161,7 +161,7 @@ impl CodeInfo {
         }
     }
 
-    fn max_stacksize(&self) -> u32 {
+    fn max_stackdepth(&self) -> u32 {
         let mut maxdepth = 0u32;
         let mut stack = Vec::with_capacity(self.blocks.len());
         let mut startdepths = vec![u32::MAX; self.blocks.len()];

From 75b886bcd6e6f2a2ff1b9fb5263f51b90391cc27 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Thu, 11 Aug 2022 01:00:46 +0900
Subject: [PATCH 243/459] enhance dis output to print line numbers

---
 ...ompiler_core__compile__tests__if_ands.snap |  17 +-
 ...mpiler_core__compile__tests__if_mixed.snap |  21 +--
 ...compiler_core__compile__tests__if_ors.snap |  17 +-
 ...pile__tests__nested_double_async_with.snap | 154 ++++++++++--------
 4 files changed, 111 insertions(+), 98 deletions(-)

diff --git a/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap b/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap
index d3344e3d..d80f10df 100644
--- a/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap
+++ b/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap
@@ -2,12 +2,13 @@
 source: compiler/src/compile.rs
 expression: "compile_exec(\"\\\nif True and False and False:\n    pass\n\")"
 ---
-                 0 LoadConst            (True)
-                 1 JumpIfFalse          (6)
-                 2 LoadConst            (False)
-                 3 JumpIfFalse          (6)
-                 4 LoadConst            (False)
-                 5 JumpIfFalse          (6)
-          >>     6 LoadConst            (None)
-                 7 ReturnValue
+  1           0 LoadConst            (True)
+              1 JumpIfFalse          (6)
+              2 LoadConst            (False)
+              3 JumpIfFalse          (6)
+              4 LoadConst            (False)
+              5 JumpIfFalse          (6)
+
+  2     >>    6 LoadConst            (None)
+              7 ReturnValue
 
diff --git a/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap b/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap
index 356067af..0a9175bb 100644
--- a/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap
+++ b/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap
@@ -2,14 +2,15 @@
 source: compiler/src/compile.rs
 expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n    pass\n\")"
 ---
-                 0 LoadConst            (True)
-                 1 JumpIfFalse          (4)
-                 2 LoadConst            (False)
-                 3 JumpIfTrue           (8)
-          >>     4 LoadConst            (False)
-                 5 JumpIfFalse          (8)
-                 6 LoadConst            (True)
-                 7 JumpIfFalse          (8)
-          >>     8 LoadConst            (None)
-                 9 ReturnValue
+  1           0 LoadConst            (True)
+              1 JumpIfFalse          (4)
+              2 LoadConst            (False)
+              3 JumpIfTrue           (8)
+        >>    4 LoadConst            (False)
+              5 JumpIfFalse          (8)
+              6 LoadConst            (True)
+              7 JumpIfFalse          (8)
+
+  2     >>    8 LoadConst            (None)
+              9 ReturnValue
 
diff --git a/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap b/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap
index 4463c3bd..4b812639 100644
--- a/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap
+++ b/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap
@@ -2,12 +2,13 @@
 source: compiler/src/compile.rs
 expression: "compile_exec(\"\\\nif True or False or False:\n    pass\n\")"
 ---
-                 0 LoadConst            (True)
-                 1 JumpIfTrue           (6)
-                 2 LoadConst            (False)
-                 3 JumpIfTrue           (6)
-                 4 LoadConst            (False)
-                 5 JumpIfFalse          (6)
-          >>     6 LoadConst            (None)
-                 7 ReturnValue
+  1           0 LoadConst            (True)
+              1 JumpIfTrue           (6)
+              2 LoadConst            (False)
+              3 JumpIfTrue           (6)
+              4 LoadConst            (False)
+              5 JumpIfFalse          (6)
+
+  2     >>    6 LoadConst            (None)
+              7 ReturnValue
 
diff --git a/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap b/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
index adc63bdf..79a1a86a 100644
--- a/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
+++ b/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
@@ -1,77 +1,87 @@
 ---
 source: compiler/src/compile.rs
 expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n    with self.subTest(type=type(stop_exc)):\n        try:\n            async with woohoo():\n                raise stop_exc\n        except Exception as ex:\n            self.assertIs(ex, stop_exc)\n        else:\n            self.fail(f'{stop_exc} was suppressed')\n\")"
-
 ---
-                 0 SetupLoop            (69)
-                 1 LoadNameAny          (0, StopIteration)
-                 2 LoadConst            ("spam")
-                 3 CallFunctionPositional (1)
-                 4 LoadNameAny          (1, StopAsyncIteration)
-                 5 LoadConst            ("ham")
-                 6 CallFunctionPositional (1)
-                 7 BuildTuple           (2, false)
-                 8 GetIter
-          >>     9 ForIter              (68)
-                10 StoreLocal           (2, stop_exc)
-                11 LoadNameAny          (3, self)
-                12 LoadMethod           (subTest)
-                13 LoadNameAny          (5, type)
-                14 LoadNameAny          (2, stop_exc)
-                15 CallFunctionPositional (1)
-                16 LoadConst            (("type"))
-                17 CallMethodKeyword    (1)
-                18 SetupWith            (65)
-                19 Pop
-                20 SetupExcept          (40)
-                21 LoadNameAny          (6, woohoo)
-                22 CallFunctionPositional (0)
-                23 BeforeAsyncWith
-                24 GetAwaitable
-                25 LoadConst            (None)
-                26 YieldFrom
-                27 SetupAsyncWith       (33)
-                28 Pop
-                29 LoadNameAny          (2, stop_exc)
-                30 Raise                (Raise)
-                31 PopBlock
-                32 EnterFinally
-          >>    33 WithCleanupStart
-                34 GetAwaitable
-                35 LoadConst            (None)
-                36 YieldFrom
-                37 WithCleanupFinish
-                38 PopBlock
-                39 Jump                 (54)
-          >>    40 Duplicate
-                41 LoadNameAny          (7, Exception)
-                42 TestOperation        (ExceptionMatch)
-                43 JumpIfFalse          (53)
-                44 StoreLocal           (8, ex)
-                45 LoadNameAny          (3, self)
-                46 LoadMethod           (assertIs)
-                47 LoadNameAny          (8, ex)
-                48 LoadNameAny          (2, stop_exc)
-                49 CallMethodPositional (2)
-                50 Pop
-                51 PopException
-                52 Jump                 (63)
-          >>    53 Raise                (Reraise)
-          >>    54 LoadNameAny          (3, self)
-                55 LoadMethod           (fail)
-                56 LoadConst            ("")
-                57 LoadNameAny          (2, stop_exc)
-                58 FormatValue          (None)
-                59 LoadConst            (" was suppressed")
-                60 BuildString          (2)
-                61 CallMethodPositional (1)
-                62 Pop
-          >>    63 PopBlock
-                64 EnterFinally
-          >>    65 WithCleanupStart
-                66 WithCleanupFinish
-                67 Jump                 (9)
-          >>    68 PopBlock
-          >>    69 LoadConst            (None)
-                70 ReturnValue
+  1           0 SetupLoop            (69)
+              1 LoadNameAny          (0, StopIteration)
+              2 LoadConst            ("spam")
+              3 CallFunctionPositional (1)
+              4 LoadNameAny          (1, StopAsyncIteration)
+              5 LoadConst            ("ham")
+              6 CallFunctionPositional (1)
+              7 BuildTuple           (2, false)
+              8 GetIter
+        >>    9 ForIter              (68)
+             10 StoreLocal           (2, stop_exc)
+
+  2          11 LoadNameAny          (3, self)
+             12 LoadMethod           (subTest)
+             13 LoadNameAny          (5, type)
+             14 LoadNameAny          (2, stop_exc)
+             15 CallFunctionPositional (1)
+             16 LoadConst            (("type"))
+             17 CallMethodKeyword    (1)
+             18 SetupWith            (65)
+             19 Pop
+
+  3          20 SetupExcept          (40)
+
+  4          21 LoadNameAny          (6, woohoo)
+             22 CallFunctionPositional (0)
+             23 BeforeAsyncWith
+             24 GetAwaitable
+             25 LoadConst            (None)
+             26 YieldFrom
+             27 SetupAsyncWith       (33)
+             28 Pop
+
+  5          29 LoadNameAny          (2, stop_exc)
+             30 Raise                (Raise)
+
+  4          31 PopBlock
+             32 EnterFinally
+        >>   33 WithCleanupStart
+             34 GetAwaitable
+             35 LoadConst            (None)
+             36 YieldFrom
+             37 WithCleanupFinish
+             38 PopBlock
+             39 Jump                 (54)
+        >>   40 Duplicate
+
+  6          41 LoadNameAny          (7, Exception)
+             42 TestOperation        (ExceptionMatch)
+             43 JumpIfFalse          (53)
+             44 StoreLocal           (8, ex)
+
+  7          45 LoadNameAny          (3, self)
+             46 LoadMethod           (assertIs)
+             47 LoadNameAny          (8, ex)
+             48 LoadNameAny          (2, stop_exc)
+             49 CallMethodPositional (2)
+             50 Pop
+             51 PopException
+             52 Jump                 (63)
+        >>   53 Raise                (Reraise)
+
+  9     >>   54 LoadNameAny          (3, self)
+             55 LoadMethod           (fail)
+             56 LoadConst            ("")
+
+  1          57 LoadNameAny          (2, stop_exc)
+             58 FormatValue          (None)
+
+  9          59 LoadConst            (" was suppressed")
+             60 BuildString          (2)
+             61 CallMethodPositional (1)
+             62 Pop
+
+  2     >>   63 PopBlock
+             64 EnterFinally
+        >>   65 WithCleanupStart
+             66 WithCleanupFinish
+             67 Jump                 (9)
+        >>   68 PopBlock
+        >>   69 LoadConst            (None)
+             70 ReturnValue
 

From a9306971ae68bf2eb3dfead38fae4558979fb733 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Thu, 11 Aug 2022 01:53:49 +0900
Subject: [PATCH 244/459] Fix with-statement location

---
 src/compile.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/compile.rs b/src/compile.rs
index f183433e..184f4108 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1393,12 +1393,15 @@ impl Compiler {
         body: &[ast::Stmt],
         is_async: bool,
     ) -> CompileResult<()> {
+        let with_location = self.current_source_location;
+
         let end_blocks = items
             .iter()
             .map(|item| {
                 let end_block = self.new_block();
                 self.compile_expression(&item.context_expr)?;
 
+                self.set_source_location(with_location);
                 if is_async {
                     self.emit(Instruction::BeforeAsyncWith);
                     self.emit(Instruction::GetAwaitable);
@@ -1411,6 +1414,7 @@ impl Compiler {
 
                 match &item.optional_vars {
                     Some(var) => {
+                        self.set_source_location(var.location);
                         self.compile_store(var)?;
                     }
                     None => {
@@ -1425,6 +1429,7 @@ impl Compiler {
 
         // sort of "stack up" the layers of with blocks:
         // with a, b: body -> start_with(a) start_with(b) body() end_with(b) end_with(a)
+        self.set_source_location(with_location);
         for end_block in end_blocks.into_iter().rev() {
             self.emit(Instruction::PopBlock);
             self.emit(Instruction::EnterFinally);

From fa6c66272ee49b3b4e59f1b47c1a3979efaba993 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Thu, 11 Aug 2022 03:39:50 +0900
Subject: [PATCH 245/459] compile_with as recursive form

---
 src/compile.rs | 88 ++++++++++++++++++++++++++++----------------------
 src/error.rs   |  8 +++++
 2 files changed, 57 insertions(+), 39 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 184f4108..8c8438ae 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -1395,57 +1395,67 @@ impl Compiler {
     ) -> CompileResult<()> {
         let with_location = self.current_source_location;
 
-        let end_blocks = items
-            .iter()
-            .map(|item| {
-                let end_block = self.new_block();
-                self.compile_expression(&item.context_expr)?;
-
-                self.set_source_location(with_location);
-                if is_async {
-                    self.emit(Instruction::BeforeAsyncWith);
-                    self.emit(Instruction::GetAwaitable);
-                    self.emit_constant(ConstantData::None);
-                    self.emit(Instruction::YieldFrom);
-                    self.emit(Instruction::SetupAsyncWith { end: end_block });
-                } else {
-                    self.emit(Instruction::SetupWith { end: end_block });
-                }
+        let (item, items) = if let Some(parts) = items.split_first() {
+            parts
+        } else {
+            return Err(self.error(CompileErrorType::EmptyWithItems));
+        };
 
-                match &item.optional_vars {
-                    Some(var) => {
-                        self.set_source_location(var.location);
-                        self.compile_store(var)?;
-                    }
-                    None => {
-                        self.emit(Instruction::Pop);
-                    }
+        let final_block = {
+            let final_block = self.new_block();
+            self.compile_expression(&item.context_expr)?;
+
+            self.set_source_location(with_location);
+            if is_async {
+                self.emit(Instruction::BeforeAsyncWith);
+                self.emit(Instruction::GetAwaitable);
+                self.emit_constant(ConstantData::None);
+                self.emit(Instruction::YieldFrom);
+                self.emit(Instruction::SetupAsyncWith { end: final_block });
+            } else {
+                self.emit(Instruction::SetupWith { end: final_block });
+            }
+
+            match &item.optional_vars {
+                Some(var) => {
+                    self.set_source_location(var.location);
+                    self.compile_store(var)?;
+                }
+                None => {
+                    self.emit(Instruction::Pop);
                 }
-                Ok(end_block)
-            })
-            .collect::<CompileResult<Vec<_>>>()?;
+            }
+            final_block
+        };
 
-        self.compile_statements(body)?;
+        if items.is_empty() {
+            if body.is_empty() {
+                return Err(self.error(CompileErrorType::EmptyWithBody));
+            }
+            self.compile_statements(body)?;
+        } else {
+            self.set_source_location(with_location);
+            self.compile_with(items, body, is_async)?;
+        }
 
         // sort of "stack up" the layers of with blocks:
         // with a, b: body -> start_with(a) start_with(b) body() end_with(b) end_with(a)
         self.set_source_location(with_location);
-        for end_block in end_blocks.into_iter().rev() {
-            self.emit(Instruction::PopBlock);
-            self.emit(Instruction::EnterFinally);
+        self.emit(Instruction::PopBlock);
 
-            self.switch_to_block(end_block);
-            self.emit(Instruction::WithCleanupStart);
+        self.emit(Instruction::EnterFinally);
 
-            if is_async {
-                self.emit(Instruction::GetAwaitable);
-                self.emit_constant(ConstantData::None);
-                self.emit(Instruction::YieldFrom);
-            }
+        self.switch_to_block(final_block);
+        self.emit(Instruction::WithCleanupStart);
 
-            self.emit(Instruction::WithCleanupFinish);
+        if is_async {
+            self.emit(Instruction::GetAwaitable);
+            self.emit_constant(ConstantData::None);
+            self.emit(Instruction::YieldFrom);
         }
 
+        self.emit(Instruction::WithCleanupFinish);
+
         Ok(())
     }
 
diff --git a/src/error.rs b/src/error.rs
index 38740aa7..1d2c1c2f 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -35,6 +35,8 @@ pub enum CompileErrorType {
     InvalidFutureFeature(String),
     FunctionImportStar,
     TooManyStarUnpack,
+    EmptyWithItems,
+    EmptyWithBody,
 }
 
 impl fmt::Display for CompileErrorType {
@@ -70,6 +72,12 @@ impl fmt::Display for CompileErrorType {
             CompileErrorType::TooManyStarUnpack => {
                 write!(f, "too many expressions in star-unpacking assignment")
             }
+            CompileErrorType::EmptyWithItems => {
+                write!(f, "empty items on With")
+            }
+            CompileErrorType::EmptyWithBody => {
+                write!(f, "empty body on With")
+            }
         }
     }
 }

From c476c276ab50f102e5715524a3782efd2671f7d6 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Fri, 12 Aug 2022 08:35:34 +0900
Subject: [PATCH 246/459] replace try_parse!() macro to closure

---
 porcelain/src/lib.rs | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/porcelain/src/lib.rs b/porcelain/src/lib.rs
index e593188a..c14c0ba3 100644
--- a/porcelain/src/lib.rs
+++ b/porcelain/src/lib.rs
@@ -99,21 +99,14 @@ pub fn compile_symtable(
     mode: compile::Mode,
     source_path: &str,
 ) -> Result<symboltable::SymbolTable, CompileError> {
-    macro_rules! try_parse {
-        ($x:expr) => {
-            match $x {
-                Ok(x) => x,
-                Err(e) => return Err(CompileError::from_parse(e, source, source_path.to_owned())),
-            }
-        };
-    }
+    let parse_err = |e| CompileError::from_parse(e, source, source_path.to_owned());
     let res = match mode {
         compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
-            let ast = try_parse!(parser::parse_program(source));
+            let ast = parser::parse_program(source).map_err(parse_err)?;
             symboltable::make_symbol_table(&ast)
         }
         compile::Mode::Eval => {
-            let expr = try_parse!(parser::parse_expression(source));
+            let expr = parser::parse_expression(source).map_err(parse_err)?;
             symboltable::make_symbol_table_expr(&expr)
         }
     };

From 308c82a9354adbd20c22771da011685bedb3e70d Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Fri, 12 Aug 2022 03:48:14 +0900
Subject: [PATCH 247/459] update asdl_rs.py for updated Python.asdl

---
 src/compile.rs     | 75 +++++++++++++++++-----------------------------
 src/symboltable.rs |  2 +-
 2 files changed, 28 insertions(+), 49 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index 8c8438ae..efb5e087 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -552,6 +552,7 @@ impl Compiler {
             Import { names } => {
                 // import a, b, c as d
                 for name in names {
+                    let name = &name.node;
                     self.emit_constant(ConstantData::Integer {
                         value: num_traits::Zero::zero(),
                     });
@@ -574,7 +575,7 @@ impl Compiler {
                 module,
                 names,
             } => {
-                let import_star = names.iter().any(|n| n.name == "*");
+                let import_star = names.iter().any(|n| n.node.name == "*");
 
                 let from_list = if import_star {
                     if self.ctx.in_func() {
@@ -588,7 +589,7 @@ impl Compiler {
                     names
                         .iter()
                         .map(|n| ConstantData::Str {
-                            value: n.name.to_owned(),
+                            value: n.node.name.to_owned(),
                         })
                         .collect()
                 };
@@ -597,7 +598,7 @@ impl Compiler {
 
                 // from .... import (*fromlist)
                 self.emit_constant(ConstantData::Integer {
-                    value: (*level).into(),
+                    value: (*level).unwrap_or(0).into(),
                 });
                 self.emit_constant(ConstantData::Tuple {
                     elements: from_list,
@@ -615,6 +616,7 @@ impl Compiler {
                     // from mod import a, b as c
 
                     for name in names {
+                        let name = &name.node;
                         let idx = self.name(&name.name);
                         // import symbol from module:
                         self.emit(Instruction::ImportFrom { idx });
@@ -881,13 +883,11 @@ impl Compiler {
 
         let mut num_kw_only_defaults = 0;
         for (kw, default) in args.kwonlyargs.iter().zip(&args.kw_defaults) {
-            if let Some(default) = default {
-                self.emit_constant(ConstantData::Str {
-                    value: kw.node.arg.clone(),
-                });
-                self.compile_expression(default)?;
-                num_kw_only_defaults += 1;
-            }
+            self.emit_constant(ConstantData::Str {
+                value: kw.node.arg.clone(),
+            });
+            self.compile_expression(default)?;
+            num_kw_only_defaults += 1;
         }
         if num_kw_only_defaults > 0 {
             self.emit(Instruction::BuildMap {
@@ -1930,49 +1930,28 @@ impl Compiler {
 
     fn compile_dict(
         &mut self,
-        keys: &[Option<Box<ast::Expr>>],
+        keys: &[ast::Expr],
         values: &[ast::Expr],
     ) -> CompileResult<()> {
         let mut size = 0;
-        let mut has_unpacking = false;
-        for (is_unpacking, subpairs) in &keys.iter().zip(values).group_by(|e| e.0.is_none()) {
-            if is_unpacking {
-                for (_, value) in subpairs {
-                    self.compile_expression(value)?;
-                    size += 1;
-                }
-                has_unpacking = true;
-            } else {
-                let mut subsize = 0;
-                for (key, value) in subpairs {
-                    if let Some(key) = key {
-                        self.compile_expression(key)?;
-                        self.compile_expression(value)?;
-                        subsize += 1;
-                    }
-                }
-                self.emit(Instruction::BuildMap {
-                    size: subsize,
-                    unpack: false,
-                    for_call: false,
-                });
-                size += 1;
-            }
-        }
-        if size == 0 {
-            self.emit(Instruction::BuildMap {
-                size,
-                unpack: false,
-                for_call: false,
-            });
+
+        let (packed_values, unpacked_values) = values.split_at(keys.len());
+        for (key, value) in keys.iter().zip(packed_values) {
+            self.compile_expression(key)?;
+            self.compile_expression(value)?;
+            size += 1;
         }
-        if size > 1 || has_unpacking {
-            self.emit(Instruction::BuildMap {
-                size,
-                unpack: true,
-                for_call: false,
-            });
+        self.emit(Instruction::BuildMap {
+            size,
+            unpack: false,
+            for_call: false,
+        });
+
+        for value in unpacked_values {
+            self.compile_expression(value)?;
+            self.emit(Instruction::DictUpdate);
         }
+
         Ok(())
     }
 
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 9a216cdf..38765634 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -1094,7 +1094,7 @@ impl SymbolTableBuilder {
     ) -> SymbolTableResult {
         // Evaluate eventual default parameters:
         self.scan_expressions(&args.defaults, ExpressionContext::Load)?;
-        for expression in args.kw_defaults.iter().flatten() {
+        for expression in args.kw_defaults.iter() {
             self.scan_expression(expression, ExpressionContext::Load)?;
         }
 

From 266b1f1aea75df12b126c3dbe66a52571cd9e082 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Fri, 12 Aug 2022 05:24:19 +0900
Subject: [PATCH 248/459] Implement new features for updated python asdl

---
 src/compile.rs     | 58 ++++++++++++++++++++++------------------------
 src/error.rs       |  4 ++++
 src/symboltable.rs | 30 ++++++++++++++++--------
 3 files changed, 52 insertions(+), 40 deletions(-)

diff --git a/src/compile.rs b/src/compile.rs
index efb5e087..6e5cbfc8 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -680,6 +680,7 @@ impl Compiler {
                 orelse,
                 ..
             } => self.compile_for(target, iter, body, orelse, true)?,
+            Match { subject, cases } => self.compile_match(subject, cases)?,
             Raise { exc, cause } => {
                 let kind = match exc {
                     Some(value) => {
@@ -881,17 +882,19 @@ impl Compiler {
             });
         }
 
-        let mut num_kw_only_defaults = 0;
-        for (kw, default) in args.kwonlyargs.iter().zip(&args.kw_defaults) {
-            self.emit_constant(ConstantData::Str {
-                value: kw.node.arg.clone(),
-            });
-            self.compile_expression(default)?;
-            num_kw_only_defaults += 1;
-        }
-        if num_kw_only_defaults > 0 {
+        if !args.kw_defaults.is_empty() {
+            let required_kw_count = args.kwonlyargs.len().saturating_sub(args.kw_defaults.len());
+            for (kw, default) in args.kwonlyargs[required_kw_count..]
+                .iter()
+                .zip(&args.kw_defaults)
+            {
+                self.emit_constant(ConstantData::Str {
+                    value: kw.node.arg.clone(),
+                });
+                self.compile_expression(default)?;
+            }
             self.emit(Instruction::BuildMap {
-                size: num_kw_only_defaults,
+                size: args.kw_defaults.len() as u32,
                 unpack: false,
                 for_call: false,
             });
@@ -901,7 +904,7 @@ impl Compiler {
         if have_defaults {
             funcflags |= bytecode::MakeFunctionFlags::DEFAULTS;
         }
-        if num_kw_only_defaults > 0 {
+        if !args.kw_defaults.is_empty() {
             funcflags |= bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS;
         }
 
@@ -1519,6 +1522,16 @@ impl Compiler {
         Ok(())
     }
 
+    fn compile_match(
+        &mut self,
+        subject: &ast::Expr,
+        cases: &[ast::MatchCase],
+    ) -> CompileResult<()> {
+        eprintln!("match subject: {subject:?}");
+        eprintln!("match cases: {cases:?}");
+        Err(self.error(CompileErrorType::NotImplementedYet))
+    }
+
     fn compile_chained_comparison(
         &mut self,
         left: &ast::Expr,
@@ -1928,11 +1941,7 @@ impl Compiler {
         Ok(())
     }
 
-    fn compile_dict(
-        &mut self,
-        keys: &[ast::Expr],
-        values: &[ast::Expr],
-    ) -> CompileResult<()> {
+    fn compile_dict(&mut self, keys: &[ast::Expr], values: &[ast::Expr]) -> CompileResult<()> {
         let mut size = 0;
 
         let (packed_values, unpacked_values) = values.split_at(keys.len());
@@ -2099,7 +2108,7 @@ impl Compiler {
                 };
                 self.compile_expression(value)?;
                 self.emit(Instruction::FormatValue {
-                    conversion: compile_conversion_flag(*conversion),
+                    conversion: (*conversion).try_into().expect("invalid conversion flag"),
                 });
             }
             Name { id, .. } => self.load_name(id)?,
@@ -2454,7 +2463,7 @@ impl Compiler {
 
         let mut loop_labels = vec![];
         for generator in generators {
-            if generator.is_async {
+            if generator.is_async > 0 {
                 unimplemented!("async for comprehensions");
             }
 
@@ -2543,7 +2552,7 @@ impl Compiler {
             return Err(self.error(CompileErrorType::InvalidFuturePlacement));
         }
         for feature in features {
-            match &*feature.name {
+            match &*feature.node.name {
                 // Python 3 features; we've already implemented them by default
                 "nested_scopes" | "generators" | "division" | "absolute_import"
                 | "with_statement" | "print_function" | "unicode_literals" => {}
@@ -2667,17 +2676,6 @@ fn compile_location(location: &ast::Location) -> bytecode::Location {
     bytecode::Location::new(location.row(), location.column())
 }
 
-fn compile_conversion_flag(
-    conversion_flag: Option<ast::ConversionFlag>,
-) -> bytecode::ConversionFlag {
-    match conversion_flag {
-        None => bytecode::ConversionFlag::None,
-        Some(ast::ConversionFlag::Ascii) => bytecode::ConversionFlag::Ascii,
-        Some(ast::ConversionFlag::Repr) => bytecode::ConversionFlag::Repr,
-        Some(ast::ConversionFlag::Str) => bytecode::ConversionFlag::Str,
-    }
-}
-
 fn compile_constant(value: &ast::Constant) -> ConstantData {
     match value {
         ast::Constant::None => ConstantData::None,
diff --git a/src/error.rs b/src/error.rs
index 1d2c1c2f..2e125b01 100644
--- a/src/error.rs
+++ b/src/error.rs
@@ -37,6 +37,7 @@ pub enum CompileErrorType {
     TooManyStarUnpack,
     EmptyWithItems,
     EmptyWithBody,
+    NotImplementedYet, // RustPython marker for unimplemented features
 }
 
 impl fmt::Display for CompileErrorType {
@@ -78,6 +79,9 @@ impl fmt::Display for CompileErrorType {
             CompileErrorType::EmptyWithBody => {
                 write!(f, "empty body on With")
             }
+            CompileErrorType::NotImplementedYet => {
+                write!(f, "RustPython does not implement this feature yet")
+            }
         }
     }
 }
diff --git a/src/symboltable.rs b/src/symboltable.rs
index 38765634..6dad4f23 100644
--- a/src/symboltable.rs
+++ b/src/symboltable.rs
@@ -639,7 +639,7 @@ impl SymbolTableBuilder {
         if let ImportFrom { module, names, .. } = &statement.node {
             if module.as_deref() == Some("__future__") {
                 for feature in names {
-                    if feature.name == "annotations" {
+                    if feature.node.name == "annotations" {
                         self.future_annotations = true;
                     }
                 }
@@ -739,13 +739,13 @@ impl SymbolTableBuilder {
             }
             Import { names } | ImportFrom { names, .. } => {
                 for name in names {
-                    if let Some(alias) = &name.asname {
+                    if let Some(alias) = &name.node.asname {
                         // `import mymodule as myalias`
                         self.register_name(alias, SymbolUsage::Imported, location)?;
                     } else {
                         // `import module`
                         self.register_name(
-                            name.name.split('.').next().unwrap(),
+                            name.node.name.split('.').next().unwrap(),
                             SymbolUsage::Imported,
                             location,
                         )?;
@@ -782,7 +782,7 @@ impl SymbolTableBuilder {
             } => {
                 // https://github.com/python/cpython/blob/main/Python/symtable.c#L1233
                 match &target.node {
-                    ast::ExprKind::Name { id, .. } if *simple => {
+                    ast::ExprKind::Name { id, .. } if *simple > 0 => {
                         self.register_name(id, SymbolUsage::AnnotationAssigned, location)?;
                     }
                     _ => {
@@ -823,6 +823,15 @@ impl SymbolTableBuilder {
                 self.scan_statements(orelse)?;
                 self.scan_statements(finalbody)?;
             }
+            Match {
+                subject: _,
+                cases: _,
+            } => {
+                return Err(SymbolTableError {
+                    error: "match expression is not implemented yet".to_owned(),
+                    location: Location::default(),
+                });
+            }
             Raise { exc, cause } => {
                 if let Some(expression) = exc {
                     self.scan_expression(expression, ExpressionContext::Load)?;
@@ -875,12 +884,13 @@ impl SymbolTableBuilder {
                 self.scan_expression(value, ExpressionContext::Load)?;
             }
             Dict { keys, values } => {
-                for (key, value) in keys.iter().zip(values) {
-                    if let Some(key) = key {
-                        self.scan_expression(key, context)?;
-                    } else {
-                        // dict unpacking marker
-                    }
+                let (packed, unpacked) = values.split_at(keys.len());
+                for (key, value) in keys.iter().zip(packed) {
+                    self.scan_expression(key, context)?;
+                    self.scan_expression(value, context)?;
+                }
+                for value in unpacked {
+                    // dict unpacking marker
                     self.scan_expression(value, context)?;
                 }
             }

From 3351b4408b5f1ed360ebc823409b0f174d6a5d8a Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 03:42:06 +0900
Subject: [PATCH 249/459] Clean up imports

---
 porcelain/src/lib.rs |  6 ++++--
 src/compile.rs       | 20 ++++++++++----------
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/porcelain/src/lib.rs b/porcelain/src/lib.rs
index c14c0ba3..fcbcfa87 100644
--- a/porcelain/src/lib.rs
+++ b/porcelain/src/lib.rs
@@ -1,7 +1,9 @@
 use rustpython_bytecode::CodeObject;
 use rustpython_compiler_core::{compile, symboltable};
-use rustpython_parser::ast::{fold::Fold, ConstantOptimizer, Location};
-use rustpython_parser::parser;
+use rustpython_parser::{
+    ast::{fold::Fold, ConstantOptimizer, Location},
+    parser,
+};
 use std::fmt;
 
 pub use compile::{CompileOpts, Mode};
diff --git a/src/compile.rs b/src/compile.rs
index 6e5cbfc8..895e8690 100644
--- a/src/compile.rs
+++ b/src/compile.rs
@@ -5,13 +5,11 @@
 //!   <https://github.com/python/cpython/blob/main/Python/compile.c>
 //!   <https://github.com/micropython/micropython/blob/master/py/compile.c>
 
-use crate::ir::{self, CodeInfo};
-pub use crate::mode::Mode;
-use crate::symboltable::{make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable};
-use crate::IndexSet;
 use crate::{
     error::{CompileError, CompileErrorType},
-    symboltable,
+    ir,
+    symboltable::{self, make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable},
+    IndexSet,
 };
 use itertools::Itertools;
 use num_complex::Complex64;
@@ -20,6 +18,8 @@ use rustpython_ast as ast;
 use rustpython_bytecode::{self as bytecode, CodeObject, ConstantData, Instruction};
 use std::borrow::Cow;
 
+pub use crate::mode::Mode;
+
 type CompileResult<T> = Result<T, CompileError>;
 
 #[derive(PartialEq, Eq, Clone, Copy)]
@@ -61,7 +61,7 @@ fn is_forbidden_name(name: &str) -> bool {
 
 /// Main structure holding the state of compilation.
 struct Compiler {
-    code_stack: Vec<CodeInfo>,
+    code_stack: Vec<ir::CodeInfo>,
     symbol_table_stack: Vec<SymbolTable>,
     source_path: String,
     current_source_location: ast::Location,
@@ -199,7 +199,7 @@ pub fn compile_expression(
 
 impl Compiler {
     fn new(opts: CompileOpts, source_path: String, code_name: String) -> Self {
-        let module_code = CodeInfo {
+        let module_code = ir::CodeInfo {
             flags: bytecode::CodeFlags::NEW_LOCALS,
             posonlyarg_count: 0,
             arg_count: 0,
@@ -278,7 +278,7 @@ impl Compiler {
 
         self.symbol_table_stack.push(table);
 
-        let info = CodeInfo {
+        let info = ir::CodeInfo {
             flags,
             posonlyarg_count,
             arg_count,
@@ -318,7 +318,7 @@ impl Compiler {
     fn _name_inner(
         &mut self,
         name: &str,
-        cache: impl FnOnce(&mut CodeInfo) -> &mut IndexSet<String>,
+        cache: impl FnOnce(&mut ir::CodeInfo) -> &mut IndexSet<String>,
     ) -> bytecode::NameIdx {
         let name = self.mangle(name);
         let cache = cache(self.current_codeinfo());
@@ -2583,7 +2583,7 @@ impl Compiler {
         self.emit(Instruction::LoadConst { idx })
     }
 
-    fn current_codeinfo(&mut self) -> &mut CodeInfo {
+    fn current_codeinfo(&mut self) -> &mut ir::CodeInfo {
         self.code_stack.last_mut().expect("no code on stack")
     }
 

From 060d153bb37e5a229d55b9d83a5c180c9b1bfced Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 04:02:00 +0900
Subject: [PATCH 250/459] reorganize compiler crates

---
 Cargo.toml                                    |   22 +-
 ast/Cargo.toml                                |   16 +
 ast/Python.asdl                               |  144 ++
 ast/asdl.py                                   |  385 ++++
 ast/asdl_rs.py                                |  726 +++++++
 ast/src/ast_gen.rs                            | 1282 ++++++++++++
 ast/src/constant.rs                           |  223 +++
 ast/src/fold_helpers.rs                       |   66 +
 ast/src/impls.rs                              |   60 +
 ast/src/lib.rs                                |   13 +
 ast/src/location.rs                           |   79 +
 ast/src/unparse.rs                            |  530 +++++
 bytecode/Cargo.toml                           |   20 +
 bytecode/src/lib.rs                           | 1395 ++++++++++++++
 codegen/Cargo.toml                            |   22 +
 {src => codegen/src}/compile.rs               |    0
 {src => codegen/src}/error.rs                 |    0
 {src => codegen/src}/ir.rs                    |    0
 codegen/src/lib.rs                            |   15 +
 {src => codegen/src}/mode.rs                  |    0
 ...thon_codegen__compile__tests__if_ands.snap |   14 +
 ...hon_codegen__compile__tests__if_mixed.snap |   16 +
 ...ython_codegen__compile__tests__if_ors.snap |   14 +
 ...pile__tests__nested_double_async_with.snap |   87 +
 ...ompiler_core__compile__tests__if_ands.snap |    0
 ...mpiler_core__compile__tests__if_mixed.snap |    0
 ...compiler_core__compile__tests__if_ors.snap |    0
 ...pile__tests__nested_double_async_with.snap |    0
 {src => codegen/src}/symboltable.rs           |    0
 parser/Cargo.toml                             |   31 +
 parser/README.md                              |   66 +
 parser/build.rs                               |  143 ++
 parser/python.lalrpop                         | 1352 +++++++++++++
 parser/src/error.rs                           |  235 +++
 parser/src/fstring.rs                         |  427 ++++
 parser/src/function.rs                        |   96 +
 parser/src/lexer.rs                           | 1712 +++++++++++++++++
 parser/src/lib.rs                             |   34 +
 parser/src/mode.rs                            |   40 +
 parser/src/parser.rs                          |  202 ++
 parser/src/python.rs                          |    3 +
 ...s__fstring_parse_selfdocumenting_base.snap |   63 +
 ...tring_parse_selfdocumenting_base_more.snap |  137 ++
 ..._fstring_parse_selfdocumenting_format.snap |   88 +
 ...__fstring__tests__parse_empty_fstring.snap |   15 +
 ...parser__fstring__tests__parse_fstring.snap |   72 +
 ..._fstring__tests__parse_fstring_equals.snap |   66 +
 ...ing__tests__parse_fstring_nested_spec.snap |  119 ++
 ...ring__tests__parse_fstring_not_equals.snap |   66 +
 ..._tests__parse_fstring_not_nested_spec.snap |   62 +
 ...sts__parse_fstring_selfdoc_prec_space.snap |   63 +
 ..._parse_fstring_selfdoc_trailing_space.snap |   63 +
 ...ring__tests__parse_fstring_yield_expr.snap |   36 +
 ...on_parser__parser__tests__parse_class.snap |  160 ++
 ...rser__tests__parse_dict_comprehension.snap |   63 +
 ...ests__parse_double_list_comprehension.snap |  178 ++
 ...on_parser__parser__tests__parse_empty.snap |    5 +
 ...parser__parser__tests__parse_f_string.snap |   39 +
 ..._tests__parse_generator_comprehension.snap |   52 +
 ...er__parser__tests__parse_if_elif_else.snap |  125 ++
 ...parse_if_else_generator_comprehension.snap |   83 +
 ...n_parser__parser__tests__parse_kwargs.snap |   78 +
 ...n_parser__parser__tests__parse_lambda.snap |   90 +
 ...rser__tests__parse_list_comprehension.snap |   52 +
 ...ed_expression_generator_comprehension.snap |   95 +
 ..._parser__parser__tests__parse_print_2.snap |   65 +
 ...ser__parser__tests__parse_print_hello.snap |   51 +
 ...n_parser__parser__tests__parse_string.snap |   28 +
 ...n_parser__parser__tests__parse_tuples.snap |   91 +
 ...tring__tests__parse_f_string_concat_1.snap |   39 +
 ...tring__tests__parse_f_string_concat_2.snap |   39 +
 ...tring__tests__parse_f_string_concat_3.snap |   63 +
 ...r__string__tests__parse_string_concat.snap |   28 +
 ...ing__tests__parse_u_f_string_concat_1.snap |   41 +
 ...ing__tests__parse_u_f_string_concat_2.snap |   41 +
 ...tring__tests__parse_u_string_concat_1.snap |   28 +
 ...tring__tests__parse_u_string_concat_2.snap |   30 +
 parser/src/string.rs                          |  143 ++
 parser/src/token.rs                           |  236 +++
 porcelain/Cargo.toml                          |   12 -
 porcelain/src/lib.rs                          |  124 --
 src/lib.rs                                    |  133 +-
 82 files changed, 12368 insertions(+), 164 deletions(-)
 create mode 100644 ast/Cargo.toml
 create mode 100644 ast/Python.asdl
 create mode 100644 ast/asdl.py
 create mode 100755 ast/asdl_rs.py
 create mode 100644 ast/src/ast_gen.rs
 create mode 100644 ast/src/constant.rs
 create mode 100644 ast/src/fold_helpers.rs
 create mode 100644 ast/src/impls.rs
 create mode 100644 ast/src/lib.rs
 create mode 100644 ast/src/location.rs
 create mode 100644 ast/src/unparse.rs
 create mode 100644 bytecode/Cargo.toml
 create mode 100644 bytecode/src/lib.rs
 create mode 100644 codegen/Cargo.toml
 rename {src => codegen/src}/compile.rs (100%)
 rename {src => codegen/src}/error.rs (100%)
 rename {src => codegen/src}/ir.rs (100%)
 create mode 100644 codegen/src/lib.rs
 rename {src => codegen/src}/mode.rs (100%)
 create mode 100644 codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap
 create mode 100644 codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap
 create mode 100644 codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap
 create mode 100644 codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap
 rename {src => codegen/src}/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap (100%)
 rename {src => codegen/src}/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap (100%)
 rename {src => codegen/src}/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap (100%)
 rename {src => codegen/src}/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap (100%)
 rename {src => codegen/src}/symboltable.rs (100%)
 create mode 100644 parser/Cargo.toml
 create mode 100644 parser/README.md
 create mode 100644 parser/build.rs
 create mode 100644 parser/python.lalrpop
 create mode 100644 parser/src/error.rs
 create mode 100644 parser/src/fstring.rs
 create mode 100644 parser/src/function.rs
 create mode 100644 parser/src/lexer.rs
 create mode 100644 parser/src/lib.rs
 create mode 100644 parser/src/mode.rs
 create mode 100644 parser/src/parser.rs
 create mode 100644 parser/src/python.rs
 create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
 create mode 100644 parser/src/string.rs
 create mode 100644 parser/src/token.rs
 delete mode 100644 porcelain/Cargo.toml
 delete mode 100644 porcelain/src/lib.rs

diff --git a/Cargo.toml b/Cargo.toml
index d2f9a5f0..df762199 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,22 +1,12 @@
 [package]
-name = "rustpython-compiler-core"
+name = "rustpython-compiler"
 version = "0.1.2"
-description = "Compiler for python code into bytecode for the rustpython VM."
+description = "A usability wrapper around rustpython-parser and rustpython-compiler-core"
 authors = ["RustPython Team"]
-repository = "https://github.com/RustPython/RustPython"
-license = "MIT"
 edition = "2021"
 
 [dependencies]
-indexmap = "1.8.1"
-itertools = "0.10.3"
-rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
-rustpython-ast = { path = "../ast", features = ["unparse"] }
-num-complex = { version = "0.4.0", features = ["serde"] }
-num-traits = "0.2.14"
-log = "0.4.16"
-ahash = "0.7.6"
-
-[dev-dependencies]
-rustpython-parser = { path = "../parser" }
-insta = "1.14.0"
+thiserror = "1.0"
+rustpython-codegen = { path = "codegen" }
+rustpython-parser = { path = "parser" }
+rustpython-bytecode = { path = "bytecode" }
diff --git a/ast/Cargo.toml b/ast/Cargo.toml
new file mode 100644
index 00000000..144ba621
--- /dev/null
+++ b/ast/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name = "rustpython-ast"
+version = "0.1.0"
+authors = ["RustPython Team"]
+edition = "2021"
+
+[features]
+default = ["constant-optimization", "fold"]
+constant-optimization = ["fold"]
+fold = []
+unparse = ["rustpython-common"]
+
+[dependencies]
+num-bigint = "0.4.3"
+rustpython-common = { path = "../../common", optional = true }
+rustpython-bytecode = { path = "../bytecode"}
diff --git a/ast/Python.asdl b/ast/Python.asdl
new file mode 100644
index 00000000..32fdc01a
--- /dev/null
+++ b/ast/Python.asdl
@@ -0,0 +1,144 @@
+-- ASDL's 4 builtin types are:
+-- identifier, int, string, constant
+
+module Python
+{
+    mod = Module(stmt* body, type_ignore* type_ignores)
+        | Interactive(stmt* body)
+        | Expression(expr body)
+        | FunctionType(expr* argtypes, expr returns)
+
+    stmt = FunctionDef(identifier name, arguments args,
+                       stmt* body, expr* decorator_list, expr? returns,
+                       string? type_comment)
+          | AsyncFunctionDef(identifier name, arguments args,
+                             stmt* body, expr* decorator_list, expr? returns,
+                             string? type_comment)
+
+          | ClassDef(identifier name,
+             expr* bases,
+             keyword* keywords,
+             stmt* body,
+             expr* decorator_list)
+          | Return(expr? value)
+
+          | Delete(expr* targets)
+          | Assign(expr* targets, expr value, string? type_comment)
+          | AugAssign(expr target, operator op, expr value)
+          -- 'simple' indicates that we annotate simple name without parens
+          | AnnAssign(expr target, expr annotation, expr? value, int simple)
+
+          -- use 'orelse' because else is a keyword in target languages
+          | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
+          | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
+          | While(expr test, stmt* body, stmt* orelse)
+          | If(expr test, stmt* body, stmt* orelse)
+          | With(withitem* items, stmt* body, string? type_comment)
+          | AsyncWith(withitem* items, stmt* body, string? type_comment)
+
+          | Match(expr subject, match_case* cases)
+
+          | Raise(expr? exc, expr? cause)
+          | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)
+          | Assert(expr test, expr? msg)
+
+          | Import(alias* names)
+          | ImportFrom(identifier? module, alias* names, int? level)
+
+          | Global(identifier* names)
+          | Nonlocal(identifier* names)
+          | Expr(expr value)
+          | Pass | Break | Continue
+
+          -- col_offset is the byte offset in the utf8 string the parser uses
+          attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
+
+          -- BoolOp() can use left & right?
+    expr = BoolOp(boolop op, expr* values)
+         | NamedExpr(expr target, expr value)
+         | BinOp(expr left, operator op, expr right)
+         | UnaryOp(unaryop op, expr operand)
+         | Lambda(arguments args, expr body)
+         | IfExp(expr test, expr body, expr orelse)
+         | Dict(expr* keys, expr* values)
+         | Set(expr* elts)
+         | ListComp(expr elt, comprehension* generators)
+         | SetComp(expr elt, comprehension* generators)
+         | DictComp(expr key, expr value, comprehension* generators)
+         | GeneratorExp(expr elt, comprehension* generators)
+         -- the grammar constrains where yield expressions can occur
+         | Await(expr value)
+         | Yield(expr? value)
+         | YieldFrom(expr value)
+         -- need sequences for compare to distinguish between
+         -- x < 4 < 3 and (x < 4) < 3
+         | Compare(expr left, cmpop* ops, expr* comparators)
+         | Call(expr func, expr* args, keyword* keywords)
+         | FormattedValue(expr value, int conversion, expr? format_spec)
+         | JoinedStr(expr* values)
+         | Constant(constant value, string? kind)
+
+         -- the following expression can appear in assignment context
+         | Attribute(expr value, identifier attr, expr_context ctx)
+         | Subscript(expr value, expr slice, expr_context ctx)
+         | Starred(expr value, expr_context ctx)
+         | Name(identifier id, expr_context ctx)
+         | List(expr* elts, expr_context ctx)
+         | Tuple(expr* elts, expr_context ctx)
+
+         -- can appear only in Subscript
+         | Slice(expr? lower, expr? upper, expr? step)
+
+          -- col_offset is the byte offset in the utf8 string the parser uses
+          attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
+
+    expr_context = Load | Store | Del
+
+    boolop = And | Or
+
+    operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift
+                 | RShift | BitOr | BitXor | BitAnd | FloorDiv
+
+    unaryop = Invert | Not | UAdd | USub
+
+    cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn
+
+    comprehension = (expr target, expr iter, expr* ifs, int is_async)
+
+    excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body)
+                    attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
+
+    arguments = (arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs,
+                 expr* kw_defaults, arg? kwarg, expr* defaults)
+
+    arg = (identifier arg, expr? annotation, string? type_comment)
+           attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
+
+    -- keyword arguments supplied to call (NULL identifier for **kwargs)
+    keyword = (identifier? arg, expr value)
+               attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
+
+    -- import name with optional 'as' alias.
+    alias = (identifier name, identifier? asname)
+             attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset)
+
+    withitem = (expr context_expr, expr? optional_vars)
+
+    match_case = (pattern pattern, expr? guard, stmt* body)
+
+    pattern = MatchValue(expr value)
+            | MatchSingleton(constant value)
+            | MatchSequence(pattern* patterns)
+            | MatchMapping(expr* keys, pattern* patterns, identifier? rest)
+            | MatchClass(expr cls, pattern* patterns, identifier* kwd_attrs, pattern* kwd_patterns)
+
+            | MatchStar(identifier? name)
+            -- The optional "rest" MatchMapping parameter handles capturing extra mapping keys
+
+            | MatchAs(pattern? pattern, identifier? name)
+            | MatchOr(pattern* patterns)
+
+             attributes (int lineno, int col_offset, int end_lineno, int end_col_offset)
+
+    type_ignore = TypeIgnore(int lineno, string tag)
+}
diff --git a/ast/asdl.py b/ast/asdl.py
new file mode 100644
index 00000000..e3e6c34d
--- /dev/null
+++ b/ast/asdl.py
@@ -0,0 +1,385 @@
+#-------------------------------------------------------------------------------
+# Parser for ASDL [1] definition files. Reads in an ASDL description and parses
+# it into an AST that describes it.
+#
+# The EBNF we're parsing here: Figure 1 of the paper [1]. Extended to support
+# modules and attributes after a product. Words starting with Capital letters
+# are terminals. Literal tokens are in "double quotes". Others are
+# non-terminals. Id is either TokenId or ConstructorId.
+#
+# module        ::= "module" Id "{" [definitions] "}"
+# definitions   ::= { TypeId "=" type }
+# type          ::= product | sum
+# product       ::= fields ["attributes" fields]
+# fields        ::= "(" { field, "," } field ")"
+# field         ::= TypeId ["?" | "*"] [Id]
+# sum           ::= constructor { "|" constructor } ["attributes" fields]
+# constructor   ::= ConstructorId [fields]
+#
+# [1] "The Zephyr Abstract Syntax Description Language" by Wang, et. al. See
+#     http://asdl.sourceforge.net/
+#-------------------------------------------------------------------------------
+from collections import namedtuple
+import re
+
+__all__ = [
+    'builtin_types', 'parse', 'AST', 'Module', 'Type', 'Constructor',
+    'Field', 'Sum', 'Product', 'VisitorBase', 'Check', 'check']
+
+# The following classes define nodes into which the ASDL description is parsed.
+# Note: this is a "meta-AST". ASDL files (such as Python.asdl) describe the AST
+# structure used by a programming language. But ASDL files themselves need to be
+# parsed. This module parses ASDL files and uses a simple AST to represent them.
+# See the EBNF at the top of the file to understand the logical connection
+# between the various node types.
+
+builtin_types = {'identifier', 'string', 'int', 'constant'}
+
+class AST:
+    def __repr__(self):
+        raise NotImplementedError
+
+class Module(AST):
+    def __init__(self, name, dfns):
+        self.name = name
+        self.dfns = dfns
+        self.types = {type.name: type.value for type in dfns}
+
+    def __repr__(self):
+        return 'Module({0.name}, {0.dfns})'.format(self)
+
+class Type(AST):
+    def __init__(self, name, value):
+        self.name = name
+        self.value = value
+
+    def __repr__(self):
+        return 'Type({0.name}, {0.value})'.format(self)
+
+class Constructor(AST):
+    def __init__(self, name, fields=None):
+        self.name = name
+        self.fields = fields or []
+
+    def __repr__(self):
+        return 'Constructor({0.name}, {0.fields})'.format(self)
+
+class Field(AST):
+    def __init__(self, type, name=None, seq=False, opt=False):
+        self.type = type
+        self.name = name
+        self.seq = seq
+        self.opt = opt
+
+    def __str__(self):
+        if self.seq:
+            extra = "*"
+        elif self.opt:
+            extra = "?"
+        else:
+            extra = ""
+
+        return "{}{} {}".format(self.type, extra, self.name)
+
+    def __repr__(self):
+        if self.seq:
+            extra = ", seq=True"
+        elif self.opt:
+            extra = ", opt=True"
+        else:
+            extra = ""
+        if self.name is None:
+            return 'Field({0.type}{1})'.format(self, extra)
+        else:
+            return 'Field({0.type}, {0.name}{1})'.format(self, extra)
+
+class Sum(AST):
+    def __init__(self, types, attributes=None):
+        self.types = types
+        self.attributes = attributes or []
+
+    def __repr__(self):
+        if self.attributes:
+            return 'Sum({0.types}, {0.attributes})'.format(self)
+        else:
+            return 'Sum({0.types})'.format(self)
+
+class Product(AST):
+    def __init__(self, fields, attributes=None):
+        self.fields = fields
+        self.attributes = attributes or []
+
+    def __repr__(self):
+        if self.attributes:
+            return 'Product({0.fields}, {0.attributes})'.format(self)
+        else:
+            return 'Product({0.fields})'.format(self)
+
+# A generic visitor for the meta-AST that describes ASDL. This can be used by
+# emitters. Note that this visitor does not provide a generic visit method, so a
+# subclass needs to define visit methods from visitModule to as deep as the
+# interesting node.
+# We also define a Check visitor that makes sure the parsed ASDL is well-formed.
+
+class VisitorBase(object):
+    """Generic tree visitor for ASTs."""
+    def __init__(self):
+        self.cache = {}
+
+    def visit(self, obj, *args):
+        klass = obj.__class__
+        meth = self.cache.get(klass)
+        if meth is None:
+            methname = "visit" + klass.__name__
+            meth = getattr(self, methname, None)
+            self.cache[klass] = meth
+        if meth:
+            try:
+                meth(obj, *args)
+            except Exception as e:
+                print("Error visiting %r: %s" % (obj, e))
+                raise
+
+class Check(VisitorBase):
+    """A visitor that checks a parsed ASDL tree for correctness.
+
+    Errors are printed and accumulated.
+    """
+    def __init__(self):
+        super(Check, self).__init__()
+        self.cons = {}
+        self.errors = 0
+        self.types = {}
+
+    def visitModule(self, mod):
+        for dfn in mod.dfns:
+            self.visit(dfn)
+
+    def visitType(self, type):
+        self.visit(type.value, str(type.name))
+
+    def visitSum(self, sum, name):
+        for t in sum.types:
+            self.visit(t, name)
+
+    def visitConstructor(self, cons, name):
+        key = str(cons.name)
+        conflict = self.cons.get(key)
+        if conflict is None:
+            self.cons[key] = name
+        else:
+            print('Redefinition of constructor {}'.format(key))
+            print('Defined in {} and {}'.format(conflict, name))
+            self.errors += 1
+        for f in cons.fields:
+            self.visit(f, key)
+
+    def visitField(self, field, name):
+        key = str(field.type)
+        l = self.types.setdefault(key, [])
+        l.append(name)
+
+    def visitProduct(self, prod, name):
+        for f in prod.fields:
+            self.visit(f, name)
+
+def check(mod):
+    """Check the parsed ASDL tree for correctness.
+
+    Return True if success. For failure, the errors are printed out and False
+    is returned.
+    """
+    v = Check()
+    v.visit(mod)
+
+    for t in v.types:
+        if t not in mod.types and not t in builtin_types:
+            v.errors += 1
+            uses = ", ".join(v.types[t])
+            print('Undefined type {}, used in {}'.format(t, uses))
+    return not v.errors
+
+# The ASDL parser itself comes next. The only interesting external interface
+# here is the top-level parse function.
+
+def parse(filename):
+    """Parse ASDL from the given file and return a Module node describing it."""
+    with open(filename, encoding="utf-8") as f:
+        parser = ASDLParser()
+        return parser.parse(f.read())
+
+# Types for describing tokens in an ASDL specification.
+class TokenKind:
+    """TokenKind is provides a scope for enumerated token kinds."""
+    (ConstructorId, TypeId, Equals, Comma, Question, Pipe, Asterisk,
+     LParen, RParen, LBrace, RBrace) = range(11)
+
+    operator_table = {
+        '=': Equals, ',': Comma,    '?': Question, '|': Pipe,    '(': LParen,
+        ')': RParen, '*': Asterisk, '{': LBrace,   '}': RBrace}
+
+Token = namedtuple('Token', 'kind value lineno')
+
+class ASDLSyntaxError(Exception):
+    def __init__(self, msg, lineno=None):
+        self.msg = msg
+        self.lineno = lineno or '<unknown>'
+
+    def __str__(self):
+        return 'Syntax error on line {0.lineno}: {0.msg}'.format(self)
+
+def tokenize_asdl(buf):
+    """Tokenize the given buffer. Yield Token objects."""
+    for lineno, line in enumerate(buf.splitlines(), 1):
+        for m in re.finditer(r'\s*(\w+|--.*|.)', line.strip()):
+            c = m.group(1)
+            if c[0].isalpha():
+                # Some kind of identifier
+                if c[0].isupper():
+                    yield Token(TokenKind.ConstructorId, c, lineno)
+                else:
+                    yield Token(TokenKind.TypeId, c, lineno)
+            elif c[:2] == '--':
+                # Comment
+                break
+            else:
+                # Operators
+                try:
+                    op_kind = TokenKind.operator_table[c]
+                except KeyError:
+                    raise ASDLSyntaxError('Invalid operator %s' % c, lineno)
+                yield Token(op_kind, c, lineno)
+
+class ASDLParser:
+    """Parser for ASDL files.
+
+    Create, then call the parse method on a buffer containing ASDL.
+    This is a simple recursive descent parser that uses tokenize_asdl for the
+    lexing.
+    """
+    def __init__(self):
+        self._tokenizer = None
+        self.cur_token = None
+
+    def parse(self, buf):
+        """Parse the ASDL in the buffer and return an AST with a Module root.
+        """
+        self._tokenizer = tokenize_asdl(buf)
+        self._advance()
+        return self._parse_module()
+
+    def _parse_module(self):
+        if self._at_keyword('module'):
+            self._advance()
+        else:
+            raise ASDLSyntaxError(
+                'Expected "module" (found {})'.format(self.cur_token.value),
+                self.cur_token.lineno)
+        name = self._match(self._id_kinds)
+        self._match(TokenKind.LBrace)
+        defs = self._parse_definitions()
+        self._match(TokenKind.RBrace)
+        return Module(name, defs)
+
+    def _parse_definitions(self):
+        defs = []
+        while self.cur_token.kind == TokenKind.TypeId:
+            typename = self._advance()
+            self._match(TokenKind.Equals)
+            type = self._parse_type()
+            defs.append(Type(typename, type))
+        return defs
+
+    def _parse_type(self):
+        if self.cur_token.kind == TokenKind.LParen:
+            # If we see a (, it's a product
+            return self._parse_product()
+        else:
+            # Otherwise it's a sum. Look for ConstructorId
+            sumlist = [Constructor(self._match(TokenKind.ConstructorId),
+                                   self._parse_optional_fields())]
+            while self.cur_token.kind  == TokenKind.Pipe:
+                # More constructors
+                self._advance()
+                sumlist.append(Constructor(
+                                self._match(TokenKind.ConstructorId),
+                                self._parse_optional_fields()))
+            return Sum(sumlist, self._parse_optional_attributes())
+
+    def _parse_product(self):
+        return Product(self._parse_fields(), self._parse_optional_attributes())
+
+    def _parse_fields(self):
+        fields = []
+        self._match(TokenKind.LParen)
+        while self.cur_token.kind == TokenKind.TypeId:
+            typename = self._advance()
+            is_seq, is_opt = self._parse_optional_field_quantifier()
+            id = (self._advance() if self.cur_token.kind in self._id_kinds
+                                  else None)
+            fields.append(Field(typename, id, seq=is_seq, opt=is_opt))
+            if self.cur_token.kind == TokenKind.RParen:
+                break
+            elif self.cur_token.kind == TokenKind.Comma:
+                self._advance()
+        self._match(TokenKind.RParen)
+        return fields
+
+    def _parse_optional_fields(self):
+        if self.cur_token.kind == TokenKind.LParen:
+            return self._parse_fields()
+        else:
+            return None
+
+    def _parse_optional_attributes(self):
+        if self._at_keyword('attributes'):
+            self._advance()
+            return self._parse_fields()
+        else:
+            return None
+
+    def _parse_optional_field_quantifier(self):
+        is_seq, is_opt = False, False
+        if self.cur_token.kind == TokenKind.Asterisk:
+            is_seq = True
+            self._advance()
+        elif self.cur_token.kind == TokenKind.Question:
+            is_opt = True
+            self._advance()
+        return is_seq, is_opt
+
+    def _advance(self):
+        """ Return the value of the current token and read the next one into
+            self.cur_token.
+        """
+        cur_val = None if self.cur_token is None else self.cur_token.value
+        try:
+            self.cur_token = next(self._tokenizer)
+        except StopIteration:
+            self.cur_token = None
+        return cur_val
+
+    _id_kinds = (TokenKind.ConstructorId, TokenKind.TypeId)
+
+    def _match(self, kind):
+        """The 'match' primitive of RD parsers.
+
+        * Verifies that the current token is of the given kind (kind can
+          be a tuple, in which the kind must match one of its members).
+        * Returns the value of the current token
+        * Reads in the next token
+        """
+        if (isinstance(kind, tuple) and self.cur_token.kind in kind or
+            self.cur_token.kind == kind
+            ):
+            value = self.cur_token.value
+            self._advance()
+            return value
+        else:
+            raise ASDLSyntaxError(
+                'Unmatched {} (found {})'.format(kind, self.cur_token.kind),
+                self.cur_token.lineno)
+
+    def _at_keyword(self, keyword):
+        return (self.cur_token.kind == TokenKind.TypeId and
+                self.cur_token.value == keyword)
diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py
new file mode 100755
index 00000000..a4c894aa
--- /dev/null
+++ b/ast/asdl_rs.py
@@ -0,0 +1,726 @@
+#! /usr/bin/env python
+"""Generate Rust code from an ASDL description."""
+
+import sys
+import json
+import textwrap
+
+from argparse import ArgumentParser
+from pathlib import Path
+
+import asdl
+
+TABSIZE = 4
+AUTOGEN_MESSAGE = "// File automatically generated by {}.\n"
+
+builtin_type_mapping = {
+    "identifier": "Ident",
+    "string": "String",
+    "int": "usize",
+    "constant": "Constant",
+}
+assert builtin_type_mapping.keys() == asdl.builtin_types
+
+
+def get_rust_type(name):
+    """Return a string for the C name of the type.
+
+    This function special cases the default types provided by asdl.
+    """
+    if name in asdl.builtin_types:
+        return builtin_type_mapping[name]
+    elif name.islower():
+        return "".join(part.capitalize() for part in name.split("_"))
+    else:
+        return name
+
+
+def is_simple(sum):
+    """Return True if a sum is a simple.
+
+    A sum is simple if its types have no fields, e.g.
+    unaryop = Invert | Not | UAdd | USub
+    """
+    for t in sum.types:
+        if t.fields:
+            return False
+    return True
+
+
+def asdl_of(name, obj):
+    if isinstance(obj, asdl.Product) or isinstance(obj, asdl.Constructor):
+        fields = ", ".join(map(str, obj.fields))
+        if fields:
+            fields = "({})".format(fields)
+        return "{}{}".format(name, fields)
+    else:
+        if is_simple(obj):
+            types = " | ".join(type.name for type in obj.types)
+        else:
+            sep = "\n{}| ".format(" " * (len(name) + 1))
+            types = sep.join(asdl_of(type.name, type) for type in obj.types)
+        return "{} = {}".format(name, types)
+
+
+class EmitVisitor(asdl.VisitorBase):
+    """Visit that emits lines"""
+
+    def __init__(self, file):
+        self.file = file
+        self.identifiers = set()
+        super(EmitVisitor, self).__init__()
+
+    def emit_identifier(self, name):
+        name = str(name)
+        if name in self.identifiers:
+            return
+        self.emit("_Py_IDENTIFIER(%s);" % name, 0)
+        self.identifiers.add(name)
+
+    def emit(self, line, depth):
+        if line:
+            line = (" " * TABSIZE * depth) + line
+        self.file.write(line + "\n")
+
+
+class TypeInfo:
+    def __init__(self, name):
+        self.name = name
+        self.has_userdata = None
+        self.children = set()
+        self.boxed = False
+
+    def __repr__(self):
+        return f"<TypeInfo: {self.name}>"
+
+    def determine_userdata(self, typeinfo, stack):
+        if self.name in stack:
+            return None
+        stack.add(self.name)
+        for child, child_seq in self.children:
+            if child in asdl.builtin_types:
+                continue
+            childinfo = typeinfo[child]
+            child_has_userdata = childinfo.determine_userdata(typeinfo, stack)
+            if self.has_userdata is None and child_has_userdata is True:
+                self.has_userdata = True
+
+        stack.remove(self.name)
+        return self.has_userdata
+
+
+class FindUserdataTypesVisitor(asdl.VisitorBase):
+    def __init__(self, typeinfo):
+        self.typeinfo = typeinfo
+        super().__init__()
+
+    def visitModule(self, mod):
+        for dfn in mod.dfns:
+            self.visit(dfn)
+        stack = set()
+        for info in self.typeinfo.values():
+            info.determine_userdata(self.typeinfo, stack)
+
+    def visitType(self, type):
+        self.typeinfo[type.name] = TypeInfo(type.name)
+        self.visit(type.value, type.name)
+
+    def visitSum(self, sum, name):
+        info = self.typeinfo[name]
+        if is_simple(sum):
+            info.has_userdata = False
+        else:
+            if len(sum.types) > 1:
+                info.boxed = True
+            if sum.attributes:
+                # attributes means Located, which has the `custom: U` field
+                info.has_userdata = True
+        for variant in sum.types:
+            self.add_children(name, variant.fields)
+
+    def visitProduct(self, product, name):
+        info = self.typeinfo[name]
+        if product.attributes:
+            # attributes means Located, which has the `custom: U` field
+            info.has_userdata = True
+        if len(product.fields) > 2:
+            info.boxed = True
+        self.add_children(name, product.fields)
+
+    def add_children(self, name, fields):
+        self.typeinfo[name].children.update((field.type, field.seq) for field in fields)
+
+
+def rust_field(field_name):
+    if field_name == "type":
+        return "type_"
+    else:
+        return field_name
+
+
+class TypeInfoEmitVisitor(EmitVisitor):
+    def __init__(self, file, typeinfo):
+        self.typeinfo = typeinfo
+        super().__init__(file)
+
+    def has_userdata(self, typ):
+        return self.typeinfo[typ].has_userdata
+
+    def get_generics(self, typ, *generics):
+        if self.has_userdata(typ):
+            return [f"<{g}>" for g in generics]
+        else:
+            return ["" for g in generics]
+
+
+class StructVisitor(TypeInfoEmitVisitor):
+    """Visitor to generate typedefs for AST."""
+
+    def visitModule(self, mod):
+        for dfn in mod.dfns:
+            self.visit(dfn)
+
+    def visitType(self, type, depth=0):
+        self.visit(type.value, type.name, depth)
+
+    def visitSum(self, sum, name, depth):
+        if is_simple(sum):
+            self.simple_sum(sum, name, depth)
+        else:
+            self.sum_with_constructors(sum, name, depth)
+
+    def emit_attrs(self, depth):
+        self.emit("#[derive(Debug, PartialEq)]", depth)
+
+    def simple_sum(self, sum, name, depth):
+        rustname = get_rust_type(name)
+        self.emit_attrs(depth)
+        self.emit(f"pub enum {rustname} {{", depth)
+        for variant in sum.types:
+            self.emit(f"{variant.name},", depth + 1)
+        self.emit("}", depth)
+        self.emit("", depth)
+
+    def sum_with_constructors(self, sum, name, depth):
+        typeinfo = self.typeinfo[name]
+        generics, generics_applied = self.get_generics(name, "U = ()", "U")
+        enumname = rustname = get_rust_type(name)
+        # all the attributes right now are for location, so if it has attrs we
+        # can just wrap it in Located<>
+        if sum.attributes:
+            enumname = rustname + "Kind"
+        self.emit_attrs(depth)
+        self.emit(f"pub enum {enumname}{generics} {{", depth)
+        for t in sum.types:
+            self.visit(t, typeinfo, depth + 1)
+        self.emit("}", depth)
+        if sum.attributes:
+            self.emit(
+                f"pub type {rustname}<U = ()> = Located<{enumname}{generics_applied}, U>;",
+                depth,
+            )
+        self.emit("", depth)
+
+    def visitConstructor(self, cons, parent, depth):
+        if cons.fields:
+            self.emit(f"{cons.name} {{", depth)
+            for f in cons.fields:
+                self.visit(f, parent, "", depth + 1)
+            self.emit("},", depth)
+        else:
+            self.emit(f"{cons.name},", depth)
+
+    def visitField(self, field, parent, vis, depth):
+        typ = get_rust_type(field.type)
+        fieldtype = self.typeinfo.get(field.type)
+        if fieldtype and fieldtype.has_userdata:
+            typ = f"{typ}<U>"
+        # don't box if we're doing Vec<T>, but do box if we're doing Vec<Option<Box<T>>>
+        if fieldtype and fieldtype.boxed and (not field.seq or field.opt):
+            typ = f"Box<{typ}>"
+        if field.opt:
+            typ = f"Option<{typ}>"
+        if field.seq:
+            typ = f"Vec<{typ}>"
+        name = rust_field(field.name)
+        self.emit(f"{vis}{name}: {typ},", depth)
+
+    def visitProduct(self, product, name, depth):
+        typeinfo = self.typeinfo[name]
+        generics, generics_applied = self.get_generics(name, "U = ()", "U")
+        dataname = rustname = get_rust_type(name)
+        if product.attributes:
+            dataname = rustname + "Data"
+        self.emit_attrs(depth)
+        has_expr = any(f.type != "identifier" for f in product.fields)
+        if has_expr:
+            datadef = f"{dataname}{generics}"
+        else:
+            datadef = dataname
+        self.emit(f"pub struct {datadef} {{", depth)
+        for f in product.fields:
+            self.visit(f, typeinfo, "pub ", depth + 1)
+        self.emit("}", depth)
+        if product.attributes:
+            # attributes should just be location info
+            if not has_expr:
+                generics_applied = ""
+            self.emit(
+                f"pub type {rustname}<U = ()> = Located<{dataname}{generics_applied}, U>;",
+                depth,
+            )
+        self.emit("", depth)
+
+
+class FoldTraitDefVisitor(TypeInfoEmitVisitor):
+    def visitModule(self, mod, depth):
+        self.emit("pub trait Fold<U> {", depth)
+        self.emit("type TargetU;", depth + 1)
+        self.emit("type Error;", depth + 1)
+        self.emit(
+            "fn map_user(&mut self, user: U) -> Result<Self::TargetU, Self::Error>;",
+            depth + 2,
+        )
+        for dfn in mod.dfns:
+            self.visit(dfn, depth + 2)
+        self.emit("}", depth)
+
+    def visitType(self, type, depth):
+        name = type.name
+        apply_u, apply_target_u = self.get_generics(name, "U", "Self::TargetU")
+        enumname = get_rust_type(name)
+        self.emit(
+            f"fn fold_{name}(&mut self, node: {enumname}{apply_u}) -> Result<{enumname}{apply_target_u}, Self::Error> {{",
+            depth,
+        )
+        self.emit(f"fold_{name}(self, node)", depth + 1)
+        self.emit("}", depth)
+
+
+class FoldImplVisitor(TypeInfoEmitVisitor):
+    def visitModule(self, mod, depth):
+        self.emit(
+            "fn fold_located<U, F: Fold<U> + ?Sized, T, MT>(folder: &mut F, node: Located<T, U>, f: impl FnOnce(&mut F, T) -> Result<MT, F::Error>) -> Result<Located<MT, F::TargetU>, F::Error> {",
+            depth,
+        )
+        self.emit(
+            "Ok(Located { custom: folder.map_user(node.custom)?, location: node.location, node: f(folder, node.node)? })",
+            depth + 1,
+        )
+        self.emit("}", depth)
+        for dfn in mod.dfns:
+            self.visit(dfn, depth)
+
+    def visitType(self, type, depth=0):
+        self.visit(type.value, type.name, depth)
+
+    def visitSum(self, sum, name, depth):
+        apply_t, apply_u, apply_target_u = self.get_generics(
+            name, "T", "U", "F::TargetU"
+        )
+        enumname = get_rust_type(name)
+        is_located = bool(sum.attributes)
+
+        self.emit(f"impl<T, U> Foldable<T, U> for {enumname}{apply_t} {{", depth)
+        self.emit(f"type Mapped = {enumname}{apply_u};", depth + 1)
+        self.emit(
+            "fn fold<F: Fold<T, TargetU = U> + ?Sized>(self, folder: &mut F) -> Result<Self::Mapped, F::Error> {",
+            depth + 1,
+        )
+        self.emit(f"folder.fold_{name}(self)", depth + 2)
+        self.emit("}", depth + 1)
+        self.emit("}", depth)
+
+        self.emit(
+            f"pub fn fold_{name}<U, F: Fold<U> + ?Sized>(#[allow(unused)] folder: &mut F, node: {enumname}{apply_u}) -> Result<{enumname}{apply_target_u}, F::Error> {{",
+            depth,
+        )
+        if is_located:
+            self.emit("fold_located(folder, node, |folder, node| {", depth)
+            enumname += "Kind"
+        self.emit("match node {", depth + 1)
+        for cons in sum.types:
+            fields_pattern = self.make_pattern(cons.fields)
+            self.emit(
+                f"{enumname}::{cons.name} {{ {fields_pattern} }} => {{", depth + 2
+            )
+            self.gen_construction(f"{enumname}::{cons.name}", cons.fields, depth + 3)
+            self.emit("}", depth + 2)
+        self.emit("}", depth + 1)
+        if is_located:
+            self.emit("})", depth)
+        self.emit("}", depth)
+
+    def visitProduct(self, product, name, depth):
+        apply_t, apply_u, apply_target_u = self.get_generics(
+            name, "T", "U", "F::TargetU"
+        )
+        structname = get_rust_type(name)
+        is_located = bool(product.attributes)
+
+        self.emit(f"impl<T, U> Foldable<T, U> for {structname}{apply_t} {{", depth)
+        self.emit(f"type Mapped = {structname}{apply_u};", depth + 1)
+        self.emit(
+            "fn fold<F: Fold<T, TargetU = U> + ?Sized>(self, folder: &mut F) -> Result<Self::Mapped, F::Error> {",
+            depth + 1,
+        )
+        self.emit(f"folder.fold_{name}(self)", depth + 2)
+        self.emit("}", depth + 1)
+        self.emit("}", depth)
+
+        self.emit(
+            f"pub fn fold_{name}<U, F: Fold<U> + ?Sized>(#[allow(unused)] folder: &mut F, node: {structname}{apply_u}) -> Result<{structname}{apply_target_u}, F::Error> {{",
+            depth,
+        )
+        if is_located:
+            self.emit("fold_located(folder, node, |folder, node| {", depth)
+            structname += "Data"
+        fields_pattern = self.make_pattern(product.fields)
+        self.emit(f"let {structname} {{ {fields_pattern} }} = node;", depth + 1)
+        self.gen_construction(structname, product.fields, depth + 1)
+        if is_located:
+            self.emit("})", depth)
+        self.emit("}", depth)
+
+    def make_pattern(self, fields):
+        return ",".join(rust_field(f.name) for f in fields)
+
+    def gen_construction(self, cons_path, fields, depth):
+        self.emit(f"Ok({cons_path} {{", depth)
+        for field in fields:
+            name = rust_field(field.name)
+            self.emit(f"{name}: Foldable::fold({name}, folder)?,", depth + 1)
+        self.emit("})", depth)
+
+
+class FoldModuleVisitor(TypeInfoEmitVisitor):
+    def visitModule(self, mod):
+        depth = 0
+        self.emit('#[cfg(feature = "fold")]', depth)
+        self.emit("pub mod fold {", depth)
+        self.emit("use super::*;", depth + 1)
+        self.emit("use crate::fold_helpers::Foldable;", depth + 1)
+        FoldTraitDefVisitor(self.file, self.typeinfo).visit(mod, depth + 1)
+        FoldImplVisitor(self.file, self.typeinfo).visit(mod, depth + 1)
+        self.emit("}", depth)
+
+
+class ClassDefVisitor(EmitVisitor):
+    def visitModule(self, mod):
+        for dfn in mod.dfns:
+            self.visit(dfn)
+
+    def visitType(self, type, depth=0):
+        self.visit(type.value, type.name, depth)
+
+    def visitSum(self, sum, name, depth):
+        structname = "NodeKind" + get_rust_type(name)
+        self.emit(
+            f'#[pyclass(module = "_ast", name = {json.dumps(name)}, base = "AstNode")]',
+            depth,
+        )
+        self.emit(f"struct {structname};", depth)
+        self.emit("#[pyclass(flags(HAS_DICT, BASETYPE))]", depth)
+        self.emit(f"impl {structname} {{}}", depth)
+        for cons in sum.types:
+            self.visit(cons, sum.attributes, structname, depth)
+
+    def visitConstructor(self, cons, attrs, base, depth):
+        self.gen_classdef(cons.name, cons.fields, attrs, depth, base)
+
+    def visitProduct(self, product, name, depth):
+        self.gen_classdef(name, product.fields, product.attributes, depth)
+
+    def gen_classdef(self, name, fields, attrs, depth, base="AstNode"):
+        structname = "Node" + get_rust_type(name)
+        self.emit(
+            f'#[pyclass(module = "_ast", name = {json.dumps(name)}, base = {json.dumps(base)})]',
+            depth,
+        )
+        self.emit(f"struct {structname};", depth)
+        self.emit("#[pyclass(flags(HAS_DICT, BASETYPE))]", depth)
+        self.emit(f"impl {structname} {{", depth)
+        self.emit(f"#[extend_class]", depth + 1)
+        self.emit(
+            "fn extend_class_with_fields(ctx: &Context, class: &'static Py<PyType>) {",
+            depth + 1,
+        )
+        fields = ",".join(
+            f"ctx.new_str(ascii!({json.dumps(f.name)})).into()" for f in fields
+        )
+        self.emit(
+            f"class.set_attr(identifier!(ctx, _fields), ctx.new_list(vec![{fields}]).into());",
+            depth + 2,
+        )
+        attrs = ",".join(
+            f"ctx.new_str(ascii!({json.dumps(attr.name)})).into()" for attr in attrs
+        )
+        self.emit(
+            f"class.set_attr(identifier!(ctx, _attributes), ctx.new_list(vec![{attrs}]).into());",
+            depth + 2,
+        )
+        self.emit("}", depth + 1)
+        self.emit("}", depth)
+
+
+class ExtendModuleVisitor(EmitVisitor):
+    def visitModule(self, mod):
+        depth = 0
+        self.emit(
+            "pub fn extend_module_nodes(vm: &VirtualMachine, module: &PyObject) {",
+            depth,
+        )
+        self.emit("extend_module!(vm, module, {", depth + 1)
+        for dfn in mod.dfns:
+            self.visit(dfn, depth + 2)
+        self.emit("})", depth + 1)
+        self.emit("}", depth)
+
+    def visitType(self, type, depth):
+        self.visit(type.value, type.name, depth)
+
+    def visitSum(self, sum, name, depth):
+        rust_name = get_rust_type(name)
+        self.emit(
+            f"{json.dumps(name)} => NodeKind{rust_name}::make_class(&vm.ctx),", depth
+        )
+        for cons in sum.types:
+            self.visit(cons, depth)
+
+    def visitConstructor(self, cons, depth):
+        self.gen_extension(cons.name, depth)
+
+    def visitProduct(self, product, name, depth):
+        self.gen_extension(name, depth)
+
+    def gen_extension(self, name, depth):
+        rust_name = get_rust_type(name)
+        self.emit(f"{json.dumps(name)} => Node{rust_name}::make_class(&vm.ctx),", depth)
+
+
+class TraitImplVisitor(EmitVisitor):
+    def visitModule(self, mod):
+        for dfn in mod.dfns:
+            self.visit(dfn)
+
+    def visitType(self, type, depth=0):
+        self.visit(type.value, type.name, depth)
+
+    def visitSum(self, sum, name, depth):
+        enumname = get_rust_type(name)
+        if sum.attributes:
+            enumname += "Kind"
+
+        self.emit(f"impl NamedNode for ast::{enumname} {{", depth)
+        self.emit(f"const NAME: &'static str = {json.dumps(name)};", depth + 1)
+        self.emit("}", depth)
+        self.emit(f"impl Node for ast::{enumname} {{", depth)
+        self.emit(
+            "fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef {", depth + 1
+        )
+        self.emit("match self {", depth + 2)
+        for variant in sum.types:
+            self.constructor_to_object(variant, enumname, depth + 3)
+        self.emit("}", depth + 2)
+        self.emit("}", depth + 1)
+        self.emit(
+            "fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> {",
+            depth + 1,
+        )
+        self.gen_sum_fromobj(sum, name, enumname, depth + 2)
+        self.emit("}", depth + 1)
+        self.emit("}", depth)
+
+    def constructor_to_object(self, cons, enumname, depth):
+        fields_pattern = self.make_pattern(cons.fields)
+        self.emit(f"ast::{enumname}::{cons.name} {{ {fields_pattern} }} => {{", depth)
+        self.make_node(cons.name, cons.fields, depth + 1)
+        self.emit("}", depth)
+
+    def visitProduct(self, product, name, depth):
+        structname = get_rust_type(name)
+        if product.attributes:
+            structname += "Data"
+
+        self.emit(f"impl NamedNode for ast::{structname} {{", depth)
+        self.emit(f"const NAME: &'static str = {json.dumps(name)};", depth + 1)
+        self.emit("}", depth)
+        self.emit(f"impl Node for ast::{structname} {{", depth)
+        self.emit(
+            "fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef {", depth + 1
+        )
+        fields_pattern = self.make_pattern(product.fields)
+        self.emit(f"let ast::{structname} {{ {fields_pattern} }} = self;", depth + 2)
+        self.make_node(name, product.fields, depth + 2)
+        self.emit("}", depth + 1)
+        self.emit(
+            "fn ast_from_object(_vm: &VirtualMachine, _object: PyObjectRef) -> PyResult<Self> {",
+            depth + 1,
+        )
+        self.gen_product_fromobj(product, name, structname, depth + 2)
+        self.emit("}", depth + 1)
+        self.emit("}", depth)
+
+    def make_node(self, variant, fields, depth):
+        rust_variant = get_rust_type(variant)
+        self.emit(
+            f"let _node = AstNode.into_ref_with_type(_vm, Node{rust_variant}::static_type().to_owned()).unwrap();",
+            depth,
+        )
+        if fields:
+            self.emit("let _dict = _node.as_object().dict().unwrap();", depth)
+        for f in fields:
+            self.emit(
+                f"_dict.set_item({json.dumps(f.name)}, {rust_field(f.name)}.ast_to_object(_vm), _vm).unwrap();",
+                depth,
+            )
+        self.emit("_node.into()", depth)
+
+    def make_pattern(self, fields):
+        return ",".join(rust_field(f.name) for f in fields)
+
+    def gen_sum_fromobj(self, sum, sumname, enumname, depth):
+        if sum.attributes:
+            self.extract_location(sumname, depth)
+
+        self.emit("let _cls = _object.class();", depth)
+        self.emit("Ok(", depth)
+        for cons in sum.types:
+            self.emit(f"if _cls.is(Node{cons.name}::static_type()) {{", depth)
+            self.gen_construction(f"{enumname}::{cons.name}", cons, sumname, depth + 1)
+            self.emit("} else", depth)
+
+        self.emit("{", depth)
+        msg = f'format!("expected some sort of {sumname}, but got {{}}",_object.repr(_vm)?)'
+        self.emit(f"return Err(_vm.new_type_error({msg}));", depth + 1)
+        self.emit("})", depth)
+
+    def gen_product_fromobj(self, product, prodname, structname, depth):
+        if product.attributes:
+            self.extract_location(prodname, depth)
+
+        self.emit("Ok(", depth)
+        self.gen_construction(structname, product, prodname, depth + 1)
+        self.emit(")", depth)
+
+    def gen_construction(self, cons_path, cons, name, depth):
+        self.emit(f"ast::{cons_path} {{", depth)
+        for field in cons.fields:
+            self.emit(
+                f"{rust_field(field.name)}: {self.decode_field(field, name)},",
+                depth + 1,
+            )
+        self.emit("}", depth)
+
+    def extract_location(self, typename, depth):
+        row = self.decode_field(asdl.Field("int", "lineno"), typename)
+        column = self.decode_field(asdl.Field("int", "col_offset"), typename)
+        self.emit(f"let _location = ast::Location::new({row}, {column});", depth)
+
+    def wrap_located_node(self, depth):
+        self.emit(f"let node = ast::Located::new(_location, node);", depth)
+
+    def decode_field(self, field, typename):
+        name = json.dumps(field.name)
+        if field.opt and not field.seq:
+            return f"get_node_field_opt(_vm, &_object, {name})?.map(|obj| Node::ast_from_object(_vm, obj)).transpose()?"
+        else:
+            return f"Node::ast_from_object(_vm, get_node_field(_vm, &_object, {name}, {json.dumps(typename)})?)?"
+
+
+class ChainOfVisitors:
+    def __init__(self, *visitors):
+        self.visitors = visitors
+
+    def visit(self, object):
+        for v in self.visitors:
+            v.visit(object)
+            v.emit("", 0)
+
+
+def write_ast_def(mod, typeinfo, f):
+    f.write(
+        textwrap.dedent(
+            """
+        #![allow(clippy::derive_partial_eq_without_eq)]
+        
+        pub use crate::constant::*;
+        pub use crate::location::Location;
+
+        type Ident = String;
+        \n
+    """
+        )
+    )
+    StructVisitor(f, typeinfo).emit_attrs(0)
+    f.write(
+        textwrap.dedent(
+            """
+        pub struct Located<T, U = ()> {
+            pub location: Location,
+            pub custom: U,
+            pub node: T,
+        }
+    
+        impl<T> Located<T> {
+            pub fn new(location: Location, node: T) -> Self {
+                Self { location, custom: (), node }
+            }
+        }
+        \n
+    """.lstrip()
+        )
+    )
+
+    c = ChainOfVisitors(StructVisitor(f, typeinfo), FoldModuleVisitor(f, typeinfo))
+    c.visit(mod)
+
+
+def write_ast_mod(mod, f):
+    f.write(
+        textwrap.dedent(
+            """
+        #![allow(clippy::all)]
+
+        use super::*;
+        use crate::common::ascii;
+
+    """
+        )
+    )
+
+    c = ChainOfVisitors(ClassDefVisitor(f), TraitImplVisitor(f), ExtendModuleVisitor(f))
+    c.visit(mod)
+
+
+def main(input_filename, ast_mod_filename, ast_def_filename, dump_module=False):
+    auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:]))
+    mod = asdl.parse(input_filename)
+    if dump_module:
+        print("Parsed Module:")
+        print(mod)
+    if not asdl.check(mod):
+        sys.exit(1)
+
+    typeinfo = {}
+    FindUserdataTypesVisitor(typeinfo).visit(mod)
+
+    with ast_def_filename.open("w") as def_file, ast_mod_filename.open("w") as mod_file:
+        def_file.write(auto_gen_msg)
+        write_ast_def(mod, typeinfo, def_file)
+
+        mod_file.write(auto_gen_msg)
+        write_ast_mod(mod, mod_file)
+
+    print(f"{ast_def_filename}, {ast_mod_filename} regenerated.")
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+    parser.add_argument("input_file", type=Path)
+    parser.add_argument("-M", "--mod-file", type=Path, required=True)
+    parser.add_argument("-D", "--def-file", type=Path, required=True)
+    parser.add_argument("-d", "--dump-module", action="store_true")
+
+    args = parser.parse_args()
+    main(args.input_file, args.mod_file, args.def_file, args.dump_module)
diff --git a/ast/src/ast_gen.rs b/ast/src/ast_gen.rs
new file mode 100644
index 00000000..fda39b84
--- /dev/null
+++ b/ast/src/ast_gen.rs
@@ -0,0 +1,1282 @@
+// File automatically generated by ast/asdl_rs.py.
+
+#![allow(clippy::derive_partial_eq_without_eq)]
+
+pub use crate::constant::*;
+pub use crate::location::Location;
+
+type Ident = String;
+
+#[derive(Debug, PartialEq)]
+pub struct Located<T, U = ()> {
+    pub location: Location,
+    pub custom: U,
+    pub node: T,
+}
+
+impl<T> Located<T> {
+    pub fn new(location: Location, node: T) -> Self {
+        Self {
+            location,
+            custom: (),
+            node,
+        }
+    }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Mod<U = ()> {
+    Module {
+        body: Vec<Stmt<U>>,
+        type_ignores: Vec<TypeIgnore>,
+    },
+    Interactive {
+        body: Vec<Stmt<U>>,
+    },
+    Expression {
+        body: Box<Expr<U>>,
+    },
+    FunctionType {
+        argtypes: Vec<Expr<U>>,
+        returns: Box<Expr<U>>,
+    },
+}
+
+#[derive(Debug, PartialEq)]
+pub enum StmtKind<U = ()> {
+    FunctionDef {
+        name: Ident,
+        args: Box<Arguments<U>>,
+        body: Vec<Stmt<U>>,
+        decorator_list: Vec<Expr<U>>,
+        returns: Option<Box<Expr<U>>>,
+        type_comment: Option<String>,
+    },
+    AsyncFunctionDef {
+        name: Ident,
+        args: Box<Arguments<U>>,
+        body: Vec<Stmt<U>>,
+        decorator_list: Vec<Expr<U>>,
+        returns: Option<Box<Expr<U>>>,
+        type_comment: Option<String>,
+    },
+    ClassDef {
+        name: Ident,
+        bases: Vec<Expr<U>>,
+        keywords: Vec<Keyword<U>>,
+        body: Vec<Stmt<U>>,
+        decorator_list: Vec<Expr<U>>,
+    },
+    Return {
+        value: Option<Box<Expr<U>>>,
+    },
+    Delete {
+        targets: Vec<Expr<U>>,
+    },
+    Assign {
+        targets: Vec<Expr<U>>,
+        value: Box<Expr<U>>,
+        type_comment: Option<String>,
+    },
+    AugAssign {
+        target: Box<Expr<U>>,
+        op: Operator,
+        value: Box<Expr<U>>,
+    },
+    AnnAssign {
+        target: Box<Expr<U>>,
+        annotation: Box<Expr<U>>,
+        value: Option<Box<Expr<U>>>,
+        simple: usize,
+    },
+    For {
+        target: Box<Expr<U>>,
+        iter: Box<Expr<U>>,
+        body: Vec<Stmt<U>>,
+        orelse: Vec<Stmt<U>>,
+        type_comment: Option<String>,
+    },
+    AsyncFor {
+        target: Box<Expr<U>>,
+        iter: Box<Expr<U>>,
+        body: Vec<Stmt<U>>,
+        orelse: Vec<Stmt<U>>,
+        type_comment: Option<String>,
+    },
+    While {
+        test: Box<Expr<U>>,
+        body: Vec<Stmt<U>>,
+        orelse: Vec<Stmt<U>>,
+    },
+    If {
+        test: Box<Expr<U>>,
+        body: Vec<Stmt<U>>,
+        orelse: Vec<Stmt<U>>,
+    },
+    With {
+        items: Vec<Withitem<U>>,
+        body: Vec<Stmt<U>>,
+        type_comment: Option<String>,
+    },
+    AsyncWith {
+        items: Vec<Withitem<U>>,
+        body: Vec<Stmt<U>>,
+        type_comment: Option<String>,
+    },
+    Match {
+        subject: Box<Expr<U>>,
+        cases: Vec<MatchCase<U>>,
+    },
+    Raise {
+        exc: Option<Box<Expr<U>>>,
+        cause: Option<Box<Expr<U>>>,
+    },
+    Try {
+        body: Vec<Stmt<U>>,
+        handlers: Vec<Excepthandler<U>>,
+        orelse: Vec<Stmt<U>>,
+        finalbody: Vec<Stmt<U>>,
+    },
+    Assert {
+        test: Box<Expr<U>>,
+        msg: Option<Box<Expr<U>>>,
+    },
+    Import {
+        names: Vec<Alias<U>>,
+    },
+    ImportFrom {
+        module: Option<Ident>,
+        names: Vec<Alias<U>>,
+        level: Option<usize>,
+    },
+    Global {
+        names: Vec<Ident>,
+    },
+    Nonlocal {
+        names: Vec<Ident>,
+    },
+    Expr {
+        value: Box<Expr<U>>,
+    },
+    Pass,
+    Break,
+    Continue,
+}
+pub type Stmt<U = ()> = Located<StmtKind<U>, U>;
+
+#[derive(Debug, PartialEq)]
+pub enum ExprKind<U = ()> {
+    BoolOp {
+        op: Boolop,
+        values: Vec<Expr<U>>,
+    },
+    NamedExpr {
+        target: Box<Expr<U>>,
+        value: Box<Expr<U>>,
+    },
+    BinOp {
+        left: Box<Expr<U>>,
+        op: Operator,
+        right: Box<Expr<U>>,
+    },
+    UnaryOp {
+        op: Unaryop,
+        operand: Box<Expr<U>>,
+    },
+    Lambda {
+        args: Box<Arguments<U>>,
+        body: Box<Expr<U>>,
+    },
+    IfExp {
+        test: Box<Expr<U>>,
+        body: Box<Expr<U>>,
+        orelse: Box<Expr<U>>,
+    },
+    Dict {
+        keys: Vec<Expr<U>>,
+        values: Vec<Expr<U>>,
+    },
+    Set {
+        elts: Vec<Expr<U>>,
+    },
+    ListComp {
+        elt: Box<Expr<U>>,
+        generators: Vec<Comprehension<U>>,
+    },
+    SetComp {
+        elt: Box<Expr<U>>,
+        generators: Vec<Comprehension<U>>,
+    },
+    DictComp {
+        key: Box<Expr<U>>,
+        value: Box<Expr<U>>,
+        generators: Vec<Comprehension<U>>,
+    },
+    GeneratorExp {
+        elt: Box<Expr<U>>,
+        generators: Vec<Comprehension<U>>,
+    },
+    Await {
+        value: Box<Expr<U>>,
+    },
+    Yield {
+        value: Option<Box<Expr<U>>>,
+    },
+    YieldFrom {
+        value: Box<Expr<U>>,
+    },
+    Compare {
+        left: Box<Expr<U>>,
+        ops: Vec<Cmpop>,
+        comparators: Vec<Expr<U>>,
+    },
+    Call {
+        func: Box<Expr<U>>,
+        args: Vec<Expr<U>>,
+        keywords: Vec<Keyword<U>>,
+    },
+    FormattedValue {
+        value: Box<Expr<U>>,
+        conversion: usize,
+        format_spec: Option<Box<Expr<U>>>,
+    },
+    JoinedStr {
+        values: Vec<Expr<U>>,
+    },
+    Constant {
+        value: Constant,
+        kind: Option<String>,
+    },
+    Attribute {
+        value: Box<Expr<U>>,
+        attr: Ident,
+        ctx: ExprContext,
+    },
+    Subscript {
+        value: Box<Expr<U>>,
+        slice: Box<Expr<U>>,
+        ctx: ExprContext,
+    },
+    Starred {
+        value: Box<Expr<U>>,
+        ctx: ExprContext,
+    },
+    Name {
+        id: Ident,
+        ctx: ExprContext,
+    },
+    List {
+        elts: Vec<Expr<U>>,
+        ctx: ExprContext,
+    },
+    Tuple {
+        elts: Vec<Expr<U>>,
+        ctx: ExprContext,
+    },
+    Slice {
+        lower: Option<Box<Expr<U>>>,
+        upper: Option<Box<Expr<U>>>,
+        step: Option<Box<Expr<U>>>,
+    },
+}
+pub type Expr<U = ()> = Located<ExprKind<U>, U>;
+
+#[derive(Debug, PartialEq)]
+pub enum ExprContext {
+    Load,
+    Store,
+    Del,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Boolop {
+    And,
+    Or,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Operator {
+    Add,
+    Sub,
+    Mult,
+    MatMult,
+    Div,
+    Mod,
+    Pow,
+    LShift,
+    RShift,
+    BitOr,
+    BitXor,
+    BitAnd,
+    FloorDiv,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Unaryop {
+    Invert,
+    Not,
+    UAdd,
+    USub,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Cmpop {
+    Eq,
+    NotEq,
+    Lt,
+    LtE,
+    Gt,
+    GtE,
+    Is,
+    IsNot,
+    In,
+    NotIn,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Comprehension<U = ()> {
+    pub target: Box<Expr<U>>,
+    pub iter: Box<Expr<U>>,
+    pub ifs: Vec<Expr<U>>,
+    pub is_async: usize,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum ExcepthandlerKind<U = ()> {
+    ExceptHandler {
+        type_: Option<Box<Expr<U>>>,
+        name: Option<Ident>,
+        body: Vec<Stmt<U>>,
+    },
+}
+pub type Excepthandler<U = ()> = Located<ExcepthandlerKind<U>, U>;
+
+#[derive(Debug, PartialEq)]
+pub struct Arguments<U = ()> {
+    pub posonlyargs: Vec<Arg<U>>,
+    pub args: Vec<Arg<U>>,
+    pub vararg: Option<Box<Arg<U>>>,
+    pub kwonlyargs: Vec<Arg<U>>,
+    pub kw_defaults: Vec<Expr<U>>,
+    pub kwarg: Option<Box<Arg<U>>>,
+    pub defaults: Vec<Expr<U>>,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct ArgData<U = ()> {
+    pub arg: Ident,
+    pub annotation: Option<Box<Expr<U>>>,
+    pub type_comment: Option<String>,
+}
+pub type Arg<U = ()> = Located<ArgData<U>, U>;
+
+#[derive(Debug, PartialEq)]
+pub struct KeywordData<U = ()> {
+    pub arg: Option<Ident>,
+    pub value: Box<Expr<U>>,
+}
+pub type Keyword<U = ()> = Located<KeywordData<U>, U>;
+
+#[derive(Debug, PartialEq)]
+pub struct AliasData {
+    pub name: Ident,
+    pub asname: Option<Ident>,
+}
+pub type Alias<U = ()> = Located<AliasData, U>;
+
+#[derive(Debug, PartialEq)]
+pub struct Withitem<U = ()> {
+    pub context_expr: Box<Expr<U>>,
+    pub optional_vars: Option<Box<Expr<U>>>,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct MatchCase<U = ()> {
+    pub pattern: Box<Pattern<U>>,
+    pub guard: Option<Box<Expr<U>>>,
+    pub body: Vec<Stmt<U>>,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum PatternKind<U = ()> {
+    MatchValue {
+        value: Box<Expr<U>>,
+    },
+    MatchSingleton {
+        value: Constant,
+    },
+    MatchSequence {
+        patterns: Vec<Pattern<U>>,
+    },
+    MatchMapping {
+        keys: Vec<Expr<U>>,
+        patterns: Vec<Pattern<U>>,
+        rest: Option<Ident>,
+    },
+    MatchClass {
+        cls: Box<Expr<U>>,
+        patterns: Vec<Pattern<U>>,
+        kwd_attrs: Vec<Ident>,
+        kwd_patterns: Vec<Pattern<U>>,
+    },
+    MatchStar {
+        name: Option<Ident>,
+    },
+    MatchAs {
+        pattern: Option<Box<Pattern<U>>>,
+        name: Option<Ident>,
+    },
+    MatchOr {
+        patterns: Vec<Pattern<U>>,
+    },
+}
+pub type Pattern<U = ()> = Located<PatternKind<U>, U>;
+
+#[derive(Debug, PartialEq)]
+pub enum TypeIgnore {
+    TypeIgnore { lineno: usize, tag: String },
+}
+
+#[cfg(feature = "fold")]
+pub mod fold {
+    use super::*;
+    use crate::fold_helpers::Foldable;
+    pub trait Fold<U> {
+        type TargetU;
+        type Error;
+        fn map_user(&mut self, user: U) -> Result<Self::TargetU, Self::Error>;
+        fn fold_mod(&mut self, node: Mod<U>) -> Result<Mod<Self::TargetU>, Self::Error> {
+            fold_mod(self, node)
+        }
+        fn fold_stmt(&mut self, node: Stmt<U>) -> Result<Stmt<Self::TargetU>, Self::Error> {
+            fold_stmt(self, node)
+        }
+        fn fold_expr(&mut self, node: Expr<U>) -> Result<Expr<Self::TargetU>, Self::Error> {
+            fold_expr(self, node)
+        }
+        fn fold_expr_context(&mut self, node: ExprContext) -> Result<ExprContext, Self::Error> {
+            fold_expr_context(self, node)
+        }
+        fn fold_boolop(&mut self, node: Boolop) -> Result<Boolop, Self::Error> {
+            fold_boolop(self, node)
+        }
+        fn fold_operator(&mut self, node: Operator) -> Result<Operator, Self::Error> {
+            fold_operator(self, node)
+        }
+        fn fold_unaryop(&mut self, node: Unaryop) -> Result<Unaryop, Self::Error> {
+            fold_unaryop(self, node)
+        }
+        fn fold_cmpop(&mut self, node: Cmpop) -> Result<Cmpop, Self::Error> {
+            fold_cmpop(self, node)
+        }
+        fn fold_comprehension(
+            &mut self,
+            node: Comprehension<U>,
+        ) -> Result<Comprehension<Self::TargetU>, Self::Error> {
+            fold_comprehension(self, node)
+        }
+        fn fold_excepthandler(
+            &mut self,
+            node: Excepthandler<U>,
+        ) -> Result<Excepthandler<Self::TargetU>, Self::Error> {
+            fold_excepthandler(self, node)
+        }
+        fn fold_arguments(
+            &mut self,
+            node: Arguments<U>,
+        ) -> Result<Arguments<Self::TargetU>, Self::Error> {
+            fold_arguments(self, node)
+        }
+        fn fold_arg(&mut self, node: Arg<U>) -> Result<Arg<Self::TargetU>, Self::Error> {
+            fold_arg(self, node)
+        }
+        fn fold_keyword(
+            &mut self,
+            node: Keyword<U>,
+        ) -> Result<Keyword<Self::TargetU>, Self::Error> {
+            fold_keyword(self, node)
+        }
+        fn fold_alias(&mut self, node: Alias<U>) -> Result<Alias<Self::TargetU>, Self::Error> {
+            fold_alias(self, node)
+        }
+        fn fold_withitem(
+            &mut self,
+            node: Withitem<U>,
+        ) -> Result<Withitem<Self::TargetU>, Self::Error> {
+            fold_withitem(self, node)
+        }
+        fn fold_match_case(
+            &mut self,
+            node: MatchCase<U>,
+        ) -> Result<MatchCase<Self::TargetU>, Self::Error> {
+            fold_match_case(self, node)
+        }
+        fn fold_pattern(
+            &mut self,
+            node: Pattern<U>,
+        ) -> Result<Pattern<Self::TargetU>, Self::Error> {
+            fold_pattern(self, node)
+        }
+        fn fold_type_ignore(&mut self, node: TypeIgnore) -> Result<TypeIgnore, Self::Error> {
+            fold_type_ignore(self, node)
+        }
+    }
+    fn fold_located<U, F: Fold<U> + ?Sized, T, MT>(
+        folder: &mut F,
+        node: Located<T, U>,
+        f: impl FnOnce(&mut F, T) -> Result<MT, F::Error>,
+    ) -> Result<Located<MT, F::TargetU>, F::Error> {
+        Ok(Located {
+            custom: folder.map_user(node.custom)?,
+            location: node.location,
+            node: f(folder, node.node)?,
+        })
+    }
+    impl<T, U> Foldable<T, U> for Mod<T> {
+        type Mapped = Mod<U>;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_mod(self)
+        }
+    }
+    pub fn fold_mod<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Mod<U>,
+    ) -> Result<Mod<F::TargetU>, F::Error> {
+        match node {
+            Mod::Module { body, type_ignores } => Ok(Mod::Module {
+                body: Foldable::fold(body, folder)?,
+                type_ignores: Foldable::fold(type_ignores, folder)?,
+            }),
+            Mod::Interactive { body } => Ok(Mod::Interactive {
+                body: Foldable::fold(body, folder)?,
+            }),
+            Mod::Expression { body } => Ok(Mod::Expression {
+                body: Foldable::fold(body, folder)?,
+            }),
+            Mod::FunctionType { argtypes, returns } => Ok(Mod::FunctionType {
+                argtypes: Foldable::fold(argtypes, folder)?,
+                returns: Foldable::fold(returns, folder)?,
+            }),
+        }
+    }
+    impl<T, U> Foldable<T, U> for Stmt<T> {
+        type Mapped = Stmt<U>;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_stmt(self)
+        }
+    }
+    pub fn fold_stmt<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Stmt<U>,
+    ) -> Result<Stmt<F::TargetU>, F::Error> {
+        fold_located(folder, node, |folder, node| match node {
+            StmtKind::FunctionDef {
+                name,
+                args,
+                body,
+                decorator_list,
+                returns,
+                type_comment,
+            } => Ok(StmtKind::FunctionDef {
+                name: Foldable::fold(name, folder)?,
+                args: Foldable::fold(args, folder)?,
+                body: Foldable::fold(body, folder)?,
+                decorator_list: Foldable::fold(decorator_list, folder)?,
+                returns: Foldable::fold(returns, folder)?,
+                type_comment: Foldable::fold(type_comment, folder)?,
+            }),
+            StmtKind::AsyncFunctionDef {
+                name,
+                args,
+                body,
+                decorator_list,
+                returns,
+                type_comment,
+            } => Ok(StmtKind::AsyncFunctionDef {
+                name: Foldable::fold(name, folder)?,
+                args: Foldable::fold(args, folder)?,
+                body: Foldable::fold(body, folder)?,
+                decorator_list: Foldable::fold(decorator_list, folder)?,
+                returns: Foldable::fold(returns, folder)?,
+                type_comment: Foldable::fold(type_comment, folder)?,
+            }),
+            StmtKind::ClassDef {
+                name,
+                bases,
+                keywords,
+                body,
+                decorator_list,
+            } => Ok(StmtKind::ClassDef {
+                name: Foldable::fold(name, folder)?,
+                bases: Foldable::fold(bases, folder)?,
+                keywords: Foldable::fold(keywords, folder)?,
+                body: Foldable::fold(body, folder)?,
+                decorator_list: Foldable::fold(decorator_list, folder)?,
+            }),
+            StmtKind::Return { value } => Ok(StmtKind::Return {
+                value: Foldable::fold(value, folder)?,
+            }),
+            StmtKind::Delete { targets } => Ok(StmtKind::Delete {
+                targets: Foldable::fold(targets, folder)?,
+            }),
+            StmtKind::Assign {
+                targets,
+                value,
+                type_comment,
+            } => Ok(StmtKind::Assign {
+                targets: Foldable::fold(targets, folder)?,
+                value: Foldable::fold(value, folder)?,
+                type_comment: Foldable::fold(type_comment, folder)?,
+            }),
+            StmtKind::AugAssign { target, op, value } => Ok(StmtKind::AugAssign {
+                target: Foldable::fold(target, folder)?,
+                op: Foldable::fold(op, folder)?,
+                value: Foldable::fold(value, folder)?,
+            }),
+            StmtKind::AnnAssign {
+                target,
+                annotation,
+                value,
+                simple,
+            } => Ok(StmtKind::AnnAssign {
+                target: Foldable::fold(target, folder)?,
+                annotation: Foldable::fold(annotation, folder)?,
+                value: Foldable::fold(value, folder)?,
+                simple: Foldable::fold(simple, folder)?,
+            }),
+            StmtKind::For {
+                target,
+                iter,
+                body,
+                orelse,
+                type_comment,
+            } => Ok(StmtKind::For {
+                target: Foldable::fold(target, folder)?,
+                iter: Foldable::fold(iter, folder)?,
+                body: Foldable::fold(body, folder)?,
+                orelse: Foldable::fold(orelse, folder)?,
+                type_comment: Foldable::fold(type_comment, folder)?,
+            }),
+            StmtKind::AsyncFor {
+                target,
+                iter,
+                body,
+                orelse,
+                type_comment,
+            } => Ok(StmtKind::AsyncFor {
+                target: Foldable::fold(target, folder)?,
+                iter: Foldable::fold(iter, folder)?,
+                body: Foldable::fold(body, folder)?,
+                orelse: Foldable::fold(orelse, folder)?,
+                type_comment: Foldable::fold(type_comment, folder)?,
+            }),
+            StmtKind::While { test, body, orelse } => Ok(StmtKind::While {
+                test: Foldable::fold(test, folder)?,
+                body: Foldable::fold(body, folder)?,
+                orelse: Foldable::fold(orelse, folder)?,
+            }),
+            StmtKind::If { test, body, orelse } => Ok(StmtKind::If {
+                test: Foldable::fold(test, folder)?,
+                body: Foldable::fold(body, folder)?,
+                orelse: Foldable::fold(orelse, folder)?,
+            }),
+            StmtKind::With {
+                items,
+                body,
+                type_comment,
+            } => Ok(StmtKind::With {
+                items: Foldable::fold(items, folder)?,
+                body: Foldable::fold(body, folder)?,
+                type_comment: Foldable::fold(type_comment, folder)?,
+            }),
+            StmtKind::AsyncWith {
+                items,
+                body,
+                type_comment,
+            } => Ok(StmtKind::AsyncWith {
+                items: Foldable::fold(items, folder)?,
+                body: Foldable::fold(body, folder)?,
+                type_comment: Foldable::fold(type_comment, folder)?,
+            }),
+            StmtKind::Match { subject, cases } => Ok(StmtKind::Match {
+                subject: Foldable::fold(subject, folder)?,
+                cases: Foldable::fold(cases, folder)?,
+            }),
+            StmtKind::Raise { exc, cause } => Ok(StmtKind::Raise {
+                exc: Foldable::fold(exc, folder)?,
+                cause: Foldable::fold(cause, folder)?,
+            }),
+            StmtKind::Try {
+                body,
+                handlers,
+                orelse,
+                finalbody,
+            } => Ok(StmtKind::Try {
+                body: Foldable::fold(body, folder)?,
+                handlers: Foldable::fold(handlers, folder)?,
+                orelse: Foldable::fold(orelse, folder)?,
+                finalbody: Foldable::fold(finalbody, folder)?,
+            }),
+            StmtKind::Assert { test, msg } => Ok(StmtKind::Assert {
+                test: Foldable::fold(test, folder)?,
+                msg: Foldable::fold(msg, folder)?,
+            }),
+            StmtKind::Import { names } => Ok(StmtKind::Import {
+                names: Foldable::fold(names, folder)?,
+            }),
+            StmtKind::ImportFrom {
+                module,
+                names,
+                level,
+            } => Ok(StmtKind::ImportFrom {
+                module: Foldable::fold(module, folder)?,
+                names: Foldable::fold(names, folder)?,
+                level: Foldable::fold(level, folder)?,
+            }),
+            StmtKind::Global { names } => Ok(StmtKind::Global {
+                names: Foldable::fold(names, folder)?,
+            }),
+            StmtKind::Nonlocal { names } => Ok(StmtKind::Nonlocal {
+                names: Foldable::fold(names, folder)?,
+            }),
+            StmtKind::Expr { value } => Ok(StmtKind::Expr {
+                value: Foldable::fold(value, folder)?,
+            }),
+            StmtKind::Pass {} => Ok(StmtKind::Pass {}),
+            StmtKind::Break {} => Ok(StmtKind::Break {}),
+            StmtKind::Continue {} => Ok(StmtKind::Continue {}),
+        })
+    }
+    impl<T, U> Foldable<T, U> for Expr<T> {
+        type Mapped = Expr<U>;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_expr(self)
+        }
+    }
+    pub fn fold_expr<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Expr<U>,
+    ) -> Result<Expr<F::TargetU>, F::Error> {
+        fold_located(folder, node, |folder, node| match node {
+            ExprKind::BoolOp { op, values } => Ok(ExprKind::BoolOp {
+                op: Foldable::fold(op, folder)?,
+                values: Foldable::fold(values, folder)?,
+            }),
+            ExprKind::NamedExpr { target, value } => Ok(ExprKind::NamedExpr {
+                target: Foldable::fold(target, folder)?,
+                value: Foldable::fold(value, folder)?,
+            }),
+            ExprKind::BinOp { left, op, right } => Ok(ExprKind::BinOp {
+                left: Foldable::fold(left, folder)?,
+                op: Foldable::fold(op, folder)?,
+                right: Foldable::fold(right, folder)?,
+            }),
+            ExprKind::UnaryOp { op, operand } => Ok(ExprKind::UnaryOp {
+                op: Foldable::fold(op, folder)?,
+                operand: Foldable::fold(operand, folder)?,
+            }),
+            ExprKind::Lambda { args, body } => Ok(ExprKind::Lambda {
+                args: Foldable::fold(args, folder)?,
+                body: Foldable::fold(body, folder)?,
+            }),
+            ExprKind::IfExp { test, body, orelse } => Ok(ExprKind::IfExp {
+                test: Foldable::fold(test, folder)?,
+                body: Foldable::fold(body, folder)?,
+                orelse: Foldable::fold(orelse, folder)?,
+            }),
+            ExprKind::Dict { keys, values } => Ok(ExprKind::Dict {
+                keys: Foldable::fold(keys, folder)?,
+                values: Foldable::fold(values, folder)?,
+            }),
+            ExprKind::Set { elts } => Ok(ExprKind::Set {
+                elts: Foldable::fold(elts, folder)?,
+            }),
+            ExprKind::ListComp { elt, generators } => Ok(ExprKind::ListComp {
+                elt: Foldable::fold(elt, folder)?,
+                generators: Foldable::fold(generators, folder)?,
+            }),
+            ExprKind::SetComp { elt, generators } => Ok(ExprKind::SetComp {
+                elt: Foldable::fold(elt, folder)?,
+                generators: Foldable::fold(generators, folder)?,
+            }),
+            ExprKind::DictComp {
+                key,
+                value,
+                generators,
+            } => Ok(ExprKind::DictComp {
+                key: Foldable::fold(key, folder)?,
+                value: Foldable::fold(value, folder)?,
+                generators: Foldable::fold(generators, folder)?,
+            }),
+            ExprKind::GeneratorExp { elt, generators } => Ok(ExprKind::GeneratorExp {
+                elt: Foldable::fold(elt, folder)?,
+                generators: Foldable::fold(generators, folder)?,
+            }),
+            ExprKind::Await { value } => Ok(ExprKind::Await {
+                value: Foldable::fold(value, folder)?,
+            }),
+            ExprKind::Yield { value } => Ok(ExprKind::Yield {
+                value: Foldable::fold(value, folder)?,
+            }),
+            ExprKind::YieldFrom { value } => Ok(ExprKind::YieldFrom {
+                value: Foldable::fold(value, folder)?,
+            }),
+            ExprKind::Compare {
+                left,
+                ops,
+                comparators,
+            } => Ok(ExprKind::Compare {
+                left: Foldable::fold(left, folder)?,
+                ops: Foldable::fold(ops, folder)?,
+                comparators: Foldable::fold(comparators, folder)?,
+            }),
+            ExprKind::Call {
+                func,
+                args,
+                keywords,
+            } => Ok(ExprKind::Call {
+                func: Foldable::fold(func, folder)?,
+                args: Foldable::fold(args, folder)?,
+                keywords: Foldable::fold(keywords, folder)?,
+            }),
+            ExprKind::FormattedValue {
+                value,
+                conversion,
+                format_spec,
+            } => Ok(ExprKind::FormattedValue {
+                value: Foldable::fold(value, folder)?,
+                conversion: Foldable::fold(conversion, folder)?,
+                format_spec: Foldable::fold(format_spec, folder)?,
+            }),
+            ExprKind::JoinedStr { values } => Ok(ExprKind::JoinedStr {
+                values: Foldable::fold(values, folder)?,
+            }),
+            ExprKind::Constant { value, kind } => Ok(ExprKind::Constant {
+                value: Foldable::fold(value, folder)?,
+                kind: Foldable::fold(kind, folder)?,
+            }),
+            ExprKind::Attribute { value, attr, ctx } => Ok(ExprKind::Attribute {
+                value: Foldable::fold(value, folder)?,
+                attr: Foldable::fold(attr, folder)?,
+                ctx: Foldable::fold(ctx, folder)?,
+            }),
+            ExprKind::Subscript { value, slice, ctx } => Ok(ExprKind::Subscript {
+                value: Foldable::fold(value, folder)?,
+                slice: Foldable::fold(slice, folder)?,
+                ctx: Foldable::fold(ctx, folder)?,
+            }),
+            ExprKind::Starred { value, ctx } => Ok(ExprKind::Starred {
+                value: Foldable::fold(value, folder)?,
+                ctx: Foldable::fold(ctx, folder)?,
+            }),
+            ExprKind::Name { id, ctx } => Ok(ExprKind::Name {
+                id: Foldable::fold(id, folder)?,
+                ctx: Foldable::fold(ctx, folder)?,
+            }),
+            ExprKind::List { elts, ctx } => Ok(ExprKind::List {
+                elts: Foldable::fold(elts, folder)?,
+                ctx: Foldable::fold(ctx, folder)?,
+            }),
+            ExprKind::Tuple { elts, ctx } => Ok(ExprKind::Tuple {
+                elts: Foldable::fold(elts, folder)?,
+                ctx: Foldable::fold(ctx, folder)?,
+            }),
+            ExprKind::Slice { lower, upper, step } => Ok(ExprKind::Slice {
+                lower: Foldable::fold(lower, folder)?,
+                upper: Foldable::fold(upper, folder)?,
+                step: Foldable::fold(step, folder)?,
+            }),
+        })
+    }
+    impl<T, U> Foldable<T, U> for ExprContext {
+        type Mapped = ExprContext;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_expr_context(self)
+        }
+    }
+    pub fn fold_expr_context<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: ExprContext,
+    ) -> Result<ExprContext, F::Error> {
+        match node {
+            ExprContext::Load {} => Ok(ExprContext::Load {}),
+            ExprContext::Store {} => Ok(ExprContext::Store {}),
+            ExprContext::Del {} => Ok(ExprContext::Del {}),
+        }
+    }
+    impl<T, U> Foldable<T, U> for Boolop {
+        type Mapped = Boolop;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_boolop(self)
+        }
+    }
+    pub fn fold_boolop<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Boolop,
+    ) -> Result<Boolop, F::Error> {
+        match node {
+            Boolop::And {} => Ok(Boolop::And {}),
+            Boolop::Or {} => Ok(Boolop::Or {}),
+        }
+    }
+    impl<T, U> Foldable<T, U> for Operator {
+        type Mapped = Operator;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_operator(self)
+        }
+    }
+    pub fn fold_operator<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Operator,
+    ) -> Result<Operator, F::Error> {
+        match node {
+            Operator::Add {} => Ok(Operator::Add {}),
+            Operator::Sub {} => Ok(Operator::Sub {}),
+            Operator::Mult {} => Ok(Operator::Mult {}),
+            Operator::MatMult {} => Ok(Operator::MatMult {}),
+            Operator::Div {} => Ok(Operator::Div {}),
+            Operator::Mod {} => Ok(Operator::Mod {}),
+            Operator::Pow {} => Ok(Operator::Pow {}),
+            Operator::LShift {} => Ok(Operator::LShift {}),
+            Operator::RShift {} => Ok(Operator::RShift {}),
+            Operator::BitOr {} => Ok(Operator::BitOr {}),
+            Operator::BitXor {} => Ok(Operator::BitXor {}),
+            Operator::BitAnd {} => Ok(Operator::BitAnd {}),
+            Operator::FloorDiv {} => Ok(Operator::FloorDiv {}),
+        }
+    }
+    impl<T, U> Foldable<T, U> for Unaryop {
+        type Mapped = Unaryop;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_unaryop(self)
+        }
+    }
+    pub fn fold_unaryop<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Unaryop,
+    ) -> Result<Unaryop, F::Error> {
+        match node {
+            Unaryop::Invert {} => Ok(Unaryop::Invert {}),
+            Unaryop::Not {} => Ok(Unaryop::Not {}),
+            Unaryop::UAdd {} => Ok(Unaryop::UAdd {}),
+            Unaryop::USub {} => Ok(Unaryop::USub {}),
+        }
+    }
+    impl<T, U> Foldable<T, U> for Cmpop {
+        type Mapped = Cmpop;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_cmpop(self)
+        }
+    }
+    pub fn fold_cmpop<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Cmpop,
+    ) -> Result<Cmpop, F::Error> {
+        match node {
+            Cmpop::Eq {} => Ok(Cmpop::Eq {}),
+            Cmpop::NotEq {} => Ok(Cmpop::NotEq {}),
+            Cmpop::Lt {} => Ok(Cmpop::Lt {}),
+            Cmpop::LtE {} => Ok(Cmpop::LtE {}),
+            Cmpop::Gt {} => Ok(Cmpop::Gt {}),
+            Cmpop::GtE {} => Ok(Cmpop::GtE {}),
+            Cmpop::Is {} => Ok(Cmpop::Is {}),
+            Cmpop::IsNot {} => Ok(Cmpop::IsNot {}),
+            Cmpop::In {} => Ok(Cmpop::In {}),
+            Cmpop::NotIn {} => Ok(Cmpop::NotIn {}),
+        }
+    }
+    impl<T, U> Foldable<T, U> for Comprehension<T> {
+        type Mapped = Comprehension<U>;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_comprehension(self)
+        }
+    }
+    pub fn fold_comprehension<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Comprehension<U>,
+    ) -> Result<Comprehension<F::TargetU>, F::Error> {
+        let Comprehension {
+            target,
+            iter,
+            ifs,
+            is_async,
+        } = node;
+        Ok(Comprehension {
+            target: Foldable::fold(target, folder)?,
+            iter: Foldable::fold(iter, folder)?,
+            ifs: Foldable::fold(ifs, folder)?,
+            is_async: Foldable::fold(is_async, folder)?,
+        })
+    }
+    impl<T, U> Foldable<T, U> for Excepthandler<T> {
+        type Mapped = Excepthandler<U>;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_excepthandler(self)
+        }
+    }
+    pub fn fold_excepthandler<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Excepthandler<U>,
+    ) -> Result<Excepthandler<F::TargetU>, F::Error> {
+        fold_located(folder, node, |folder, node| match node {
+            ExcepthandlerKind::ExceptHandler { type_, name, body } => {
+                Ok(ExcepthandlerKind::ExceptHandler {
+                    type_: Foldable::fold(type_, folder)?,
+                    name: Foldable::fold(name, folder)?,
+                    body: Foldable::fold(body, folder)?,
+                })
+            }
+        })
+    }
+    impl<T, U> Foldable<T, U> for Arguments<T> {
+        type Mapped = Arguments<U>;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_arguments(self)
+        }
+    }
+    pub fn fold_arguments<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Arguments<U>,
+    ) -> Result<Arguments<F::TargetU>, F::Error> {
+        let Arguments {
+            posonlyargs,
+            args,
+            vararg,
+            kwonlyargs,
+            kw_defaults,
+            kwarg,
+            defaults,
+        } = node;
+        Ok(Arguments {
+            posonlyargs: Foldable::fold(posonlyargs, folder)?,
+            args: Foldable::fold(args, folder)?,
+            vararg: Foldable::fold(vararg, folder)?,
+            kwonlyargs: Foldable::fold(kwonlyargs, folder)?,
+            kw_defaults: Foldable::fold(kw_defaults, folder)?,
+            kwarg: Foldable::fold(kwarg, folder)?,
+            defaults: Foldable::fold(defaults, folder)?,
+        })
+    }
+    impl<T, U> Foldable<T, U> for Arg<T> {
+        type Mapped = Arg<U>;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_arg(self)
+        }
+    }
+    pub fn fold_arg<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Arg<U>,
+    ) -> Result<Arg<F::TargetU>, F::Error> {
+        fold_located(folder, node, |folder, node| {
+            let ArgData {
+                arg,
+                annotation,
+                type_comment,
+            } = node;
+            Ok(ArgData {
+                arg: Foldable::fold(arg, folder)?,
+                annotation: Foldable::fold(annotation, folder)?,
+                type_comment: Foldable::fold(type_comment, folder)?,
+            })
+        })
+    }
+    impl<T, U> Foldable<T, U> for Keyword<T> {
+        type Mapped = Keyword<U>;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_keyword(self)
+        }
+    }
+    pub fn fold_keyword<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Keyword<U>,
+    ) -> Result<Keyword<F::TargetU>, F::Error> {
+        fold_located(folder, node, |folder, node| {
+            let KeywordData { arg, value } = node;
+            Ok(KeywordData {
+                arg: Foldable::fold(arg, folder)?,
+                value: Foldable::fold(value, folder)?,
+            })
+        })
+    }
+    impl<T, U> Foldable<T, U> for Alias<T> {
+        type Mapped = Alias<U>;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_alias(self)
+        }
+    }
+    pub fn fold_alias<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Alias<U>,
+    ) -> Result<Alias<F::TargetU>, F::Error> {
+        fold_located(folder, node, |folder, node| {
+            let AliasData { name, asname } = node;
+            Ok(AliasData {
+                name: Foldable::fold(name, folder)?,
+                asname: Foldable::fold(asname, folder)?,
+            })
+        })
+    }
+    impl<T, U> Foldable<T, U> for Withitem<T> {
+        type Mapped = Withitem<U>;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_withitem(self)
+        }
+    }
+    pub fn fold_withitem<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Withitem<U>,
+    ) -> Result<Withitem<F::TargetU>, F::Error> {
+        let Withitem {
+            context_expr,
+            optional_vars,
+        } = node;
+        Ok(Withitem {
+            context_expr: Foldable::fold(context_expr, folder)?,
+            optional_vars: Foldable::fold(optional_vars, folder)?,
+        })
+    }
+    impl<T, U> Foldable<T, U> for MatchCase<T> {
+        type Mapped = MatchCase<U>;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_match_case(self)
+        }
+    }
+    pub fn fold_match_case<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: MatchCase<U>,
+    ) -> Result<MatchCase<F::TargetU>, F::Error> {
+        let MatchCase {
+            pattern,
+            guard,
+            body,
+        } = node;
+        Ok(MatchCase {
+            pattern: Foldable::fold(pattern, folder)?,
+            guard: Foldable::fold(guard, folder)?,
+            body: Foldable::fold(body, folder)?,
+        })
+    }
+    impl<T, U> Foldable<T, U> for Pattern<T> {
+        type Mapped = Pattern<U>;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_pattern(self)
+        }
+    }
+    pub fn fold_pattern<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: Pattern<U>,
+    ) -> Result<Pattern<F::TargetU>, F::Error> {
+        fold_located(folder, node, |folder, node| match node {
+            PatternKind::MatchValue { value } => Ok(PatternKind::MatchValue {
+                value: Foldable::fold(value, folder)?,
+            }),
+            PatternKind::MatchSingleton { value } => Ok(PatternKind::MatchSingleton {
+                value: Foldable::fold(value, folder)?,
+            }),
+            PatternKind::MatchSequence { patterns } => Ok(PatternKind::MatchSequence {
+                patterns: Foldable::fold(patterns, folder)?,
+            }),
+            PatternKind::MatchMapping {
+                keys,
+                patterns,
+                rest,
+            } => Ok(PatternKind::MatchMapping {
+                keys: Foldable::fold(keys, folder)?,
+                patterns: Foldable::fold(patterns, folder)?,
+                rest: Foldable::fold(rest, folder)?,
+            }),
+            PatternKind::MatchClass {
+                cls,
+                patterns,
+                kwd_attrs,
+                kwd_patterns,
+            } => Ok(PatternKind::MatchClass {
+                cls: Foldable::fold(cls, folder)?,
+                patterns: Foldable::fold(patterns, folder)?,
+                kwd_attrs: Foldable::fold(kwd_attrs, folder)?,
+                kwd_patterns: Foldable::fold(kwd_patterns, folder)?,
+            }),
+            PatternKind::MatchStar { name } => Ok(PatternKind::MatchStar {
+                name: Foldable::fold(name, folder)?,
+            }),
+            PatternKind::MatchAs { pattern, name } => Ok(PatternKind::MatchAs {
+                pattern: Foldable::fold(pattern, folder)?,
+                name: Foldable::fold(name, folder)?,
+            }),
+            PatternKind::MatchOr { patterns } => Ok(PatternKind::MatchOr {
+                patterns: Foldable::fold(patterns, folder)?,
+            }),
+        })
+    }
+    impl<T, U> Foldable<T, U> for TypeIgnore {
+        type Mapped = TypeIgnore;
+        fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+            self,
+            folder: &mut F,
+        ) -> Result<Self::Mapped, F::Error> {
+            folder.fold_type_ignore(self)
+        }
+    }
+    pub fn fold_type_ignore<U, F: Fold<U> + ?Sized>(
+        #[allow(unused)] folder: &mut F,
+        node: TypeIgnore,
+    ) -> Result<TypeIgnore, F::Error> {
+        match node {
+            TypeIgnore::TypeIgnore { lineno, tag } => Ok(TypeIgnore::TypeIgnore {
+                lineno: Foldable::fold(lineno, folder)?,
+                tag: Foldable::fold(tag, folder)?,
+            }),
+        }
+    }
+}
diff --git a/ast/src/constant.rs b/ast/src/constant.rs
new file mode 100644
index 00000000..1d165594
--- /dev/null
+++ b/ast/src/constant.rs
@@ -0,0 +1,223 @@
+use num_bigint::BigInt;
+pub use rustpython_bytecode::ConversionFlag;
+
+#[derive(Debug, PartialEq)]
+pub enum Constant {
+    None,
+    Bool(bool),
+    Str(String),
+    Bytes(Vec<u8>),
+    Int(BigInt),
+    Tuple(Vec<Constant>),
+    Float(f64),
+    Complex { real: f64, imag: f64 },
+    Ellipsis,
+}
+
+impl From<String> for Constant {
+    fn from(s: String) -> Constant {
+        Self::Str(s)
+    }
+}
+impl From<Vec<u8>> for Constant {
+    fn from(b: Vec<u8>) -> Constant {
+        Self::Bytes(b)
+    }
+}
+impl From<bool> for Constant {
+    fn from(b: bool) -> Constant {
+        Self::Bool(b)
+    }
+}
+impl From<BigInt> for Constant {
+    fn from(i: BigInt) -> Constant {
+        Self::Int(i)
+    }
+}
+
+#[cfg(feature = "rustpython-common")]
+impl std::fmt::Display for Constant {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Constant::None => f.pad("None"),
+            Constant::Bool(b) => f.pad(if *b { "True" } else { "False" }),
+            Constant::Str(s) => rustpython_common::str::repr(s).fmt(f),
+            Constant::Bytes(b) => f.pad(&rustpython_common::bytes::repr(b)),
+            Constant::Int(i) => i.fmt(f),
+            Constant::Tuple(tup) => {
+                if let [elt] = &**tup {
+                    write!(f, "({},)", elt)
+                } else {
+                    f.write_str("(")?;
+                    for (i, elt) in tup.iter().enumerate() {
+                        if i != 0 {
+                            f.write_str(", ")?;
+                        }
+                        elt.fmt(f)?;
+                    }
+                    f.write_str(")")
+                }
+            }
+            Constant::Float(fp) => f.pad(&rustpython_common::float_ops::to_string(*fp)),
+            Constant::Complex { real, imag } => {
+                if *real == 0.0 {
+                    write!(f, "{}j", imag)
+                } else {
+                    write!(f, "({}{:+}j)", real, imag)
+                }
+            }
+            Constant::Ellipsis => f.pad("..."),
+        }
+    }
+}
+
+#[cfg(feature = "constant-optimization")]
+#[non_exhaustive]
+#[derive(Default)]
+pub struct ConstantOptimizer {}
+
+#[cfg(feature = "constant-optimization")]
+impl ConstantOptimizer {
+    #[inline]
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+#[cfg(feature = "constant-optimization")]
+impl<U> crate::fold::Fold<U> for ConstantOptimizer {
+    type TargetU = U;
+    type Error = std::convert::Infallible;
+    #[inline]
+    fn map_user(&mut self, user: U) -> Result<Self::TargetU, Self::Error> {
+        Ok(user)
+    }
+    fn fold_expr(&mut self, node: crate::Expr<U>) -> Result<crate::Expr<U>, Self::Error> {
+        match node.node {
+            crate::ExprKind::Tuple { elts, ctx } => {
+                let elts = elts
+                    .into_iter()
+                    .map(|x| self.fold_expr(x))
+                    .collect::<Result<Vec<_>, _>>()?;
+                let expr = if elts
+                    .iter()
+                    .all(|e| matches!(e.node, crate::ExprKind::Constant { .. }))
+                {
+                    let tuple = elts
+                        .into_iter()
+                        .map(|e| match e.node {
+                            crate::ExprKind::Constant { value, .. } => value,
+                            _ => unreachable!(),
+                        })
+                        .collect();
+                    crate::ExprKind::Constant {
+                        value: Constant::Tuple(tuple),
+                        kind: None,
+                    }
+                } else {
+                    crate::ExprKind::Tuple { elts, ctx }
+                };
+                Ok(crate::Expr {
+                    node: expr,
+                    custom: node.custom,
+                    location: node.location,
+                })
+            }
+            _ => crate::fold::fold_expr(self, node),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    #[cfg(feature = "constant-optimization")]
+    #[test]
+    fn test_constant_opt() {
+        use super::*;
+        use crate::fold::Fold;
+        use crate::*;
+
+        let location = Location::new(0, 0);
+        let custom = ();
+        let ast = Located {
+            location,
+            custom,
+            node: ExprKind::Tuple {
+                ctx: ExprContext::Load,
+                elts: vec![
+                    Located {
+                        location,
+                        custom,
+                        node: ExprKind::Constant {
+                            value: BigInt::from(1).into(),
+                            kind: None,
+                        },
+                    },
+                    Located {
+                        location,
+                        custom,
+                        node: ExprKind::Constant {
+                            value: BigInt::from(2).into(),
+                            kind: None,
+                        },
+                    },
+                    Located {
+                        location,
+                        custom,
+                        node: ExprKind::Tuple {
+                            ctx: ExprContext::Load,
+                            elts: vec![
+                                Located {
+                                    location,
+                                    custom,
+                                    node: ExprKind::Constant {
+                                        value: BigInt::from(3).into(),
+                                        kind: None,
+                                    },
+                                },
+                                Located {
+                                    location,
+                                    custom,
+                                    node: ExprKind::Constant {
+                                        value: BigInt::from(4).into(),
+                                        kind: None,
+                                    },
+                                },
+                                Located {
+                                    location,
+                                    custom,
+                                    node: ExprKind::Constant {
+                                        value: BigInt::from(5).into(),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                ],
+            },
+        };
+        let new_ast = ConstantOptimizer::new()
+            .fold_expr(ast)
+            .unwrap_or_else(|e| match e {});
+        assert_eq!(
+            new_ast,
+            Located {
+                location,
+                custom,
+                node: ExprKind::Constant {
+                    value: Constant::Tuple(vec![
+                        BigInt::from(1).into(),
+                        BigInt::from(2).into(),
+                        Constant::Tuple(vec![
+                            BigInt::from(3).into(),
+                            BigInt::from(4).into(),
+                            BigInt::from(5).into(),
+                        ])
+                    ]),
+                    kind: None
+                },
+            }
+        );
+    }
+}
diff --git a/ast/src/fold_helpers.rs b/ast/src/fold_helpers.rs
new file mode 100644
index 00000000..f15a9d5d
--- /dev/null
+++ b/ast/src/fold_helpers.rs
@@ -0,0 +1,66 @@
+use crate::constant;
+use crate::fold::Fold;
+
+pub(crate) trait Foldable<T, U> {
+    type Mapped;
+    fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self::Mapped, F::Error>;
+}
+
+impl<T, U, X> Foldable<T, U> for Vec<X>
+where
+    X: Foldable<T, U>,
+{
+    type Mapped = Vec<X::Mapped>;
+    fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self::Mapped, F::Error> {
+        self.into_iter().map(|x| x.fold(folder)).collect()
+    }
+}
+
+impl<T, U, X> Foldable<T, U> for Option<X>
+where
+    X: Foldable<T, U>,
+{
+    type Mapped = Option<X::Mapped>;
+    fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self::Mapped, F::Error> {
+        self.map(|x| x.fold(folder)).transpose()
+    }
+}
+
+impl<T, U, X> Foldable<T, U> for Box<X>
+where
+    X: Foldable<T, U>,
+{
+    type Mapped = Box<X::Mapped>;
+    fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self::Mapped, F::Error> {
+        (*self).fold(folder).map(Box::new)
+    }
+}
+
+macro_rules! simple_fold {
+    ($($t:ty),+$(,)?) => {
+        $(impl<T, U> $crate::fold_helpers::Foldable<T, U> for $t {
+            type Mapped = Self;
+            #[inline]
+            fn fold<F: Fold<T, TargetU = U> + ?Sized>(
+                self,
+                _folder: &mut F,
+            ) -> Result<Self::Mapped, F::Error> {
+                Ok(self)
+            }
+        })+
+    };
+}
+
+simple_fold!(usize, String, bool, constant::Constant);
diff --git a/ast/src/impls.rs b/ast/src/impls.rs
new file mode 100644
index 00000000..a9302887
--- /dev/null
+++ b/ast/src/impls.rs
@@ -0,0 +1,60 @@
+use crate::{Constant, ExprKind};
+
+impl<U> ExprKind<U> {
+    /// Returns a short name for the node suitable for use in error messages.
+    pub fn name(&self) -> &'static str {
+        match self {
+            ExprKind::BoolOp { .. } | ExprKind::BinOp { .. } | ExprKind::UnaryOp { .. } => {
+                "operator"
+            }
+            ExprKind::Subscript { .. } => "subscript",
+            ExprKind::Await { .. } => "await expression",
+            ExprKind::Yield { .. } | ExprKind::YieldFrom { .. } => "yield expression",
+            ExprKind::Compare { .. } => "comparison",
+            ExprKind::Attribute { .. } => "attribute",
+            ExprKind::Call { .. } => "function call",
+            ExprKind::Constant { value, .. } => match value {
+                Constant::Str(_)
+                | Constant::Int(_)
+                | Constant::Float(_)
+                | Constant::Complex { .. }
+                | Constant::Bytes(_) => "literal",
+                Constant::Tuple(_) => "tuple",
+                Constant::Bool(b) => {
+                    if *b {
+                        "True"
+                    } else {
+                        "False"
+                    }
+                }
+                Constant::None => "None",
+                Constant::Ellipsis => "ellipsis",
+            },
+            ExprKind::List { .. } => "list",
+            ExprKind::Tuple { .. } => "tuple",
+            ExprKind::Dict { .. } => "dict display",
+            ExprKind::Set { .. } => "set display",
+            ExprKind::ListComp { .. } => "list comprehension",
+            ExprKind::DictComp { .. } => "dict comprehension",
+            ExprKind::SetComp { .. } => "set comprehension",
+            ExprKind::GeneratorExp { .. } => "generator expression",
+            ExprKind::Starred { .. } => "starred",
+            ExprKind::Slice { .. } => "slice",
+            ExprKind::JoinedStr { values } => {
+                if values
+                    .iter()
+                    .any(|e| matches!(e.node, ExprKind::JoinedStr { .. }))
+                {
+                    "f-string expression"
+                } else {
+                    "literal"
+                }
+            }
+            ExprKind::FormattedValue { .. } => "f-string expression",
+            ExprKind::Name { .. } => "name",
+            ExprKind::Lambda { .. } => "lambda",
+            ExprKind::IfExp { .. } => "conditional expression",
+            ExprKind::NamedExpr { .. } => "named expression",
+        }
+    }
+}
diff --git a/ast/src/lib.rs b/ast/src/lib.rs
new file mode 100644
index 00000000..b6969986
--- /dev/null
+++ b/ast/src/lib.rs
@@ -0,0 +1,13 @@
+mod ast_gen;
+mod constant;
+#[cfg(feature = "fold")]
+mod fold_helpers;
+mod impls;
+mod location;
+#[cfg(feature = "unparse")]
+mod unparse;
+
+pub use ast_gen::*;
+pub use location::Location;
+
+pub type Suite<U = ()> = Vec<Stmt<U>>;
diff --git a/ast/src/location.rs b/ast/src/location.rs
new file mode 100644
index 00000000..830cb5fe
--- /dev/null
+++ b/ast/src/location.rs
@@ -0,0 +1,79 @@
+//! Datatypes to support source location information.
+
+use std::fmt;
+
+/// A location somewhere in the sourcecode.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub struct Location {
+    row: usize,
+    column: usize,
+}
+
+impl fmt::Display for Location {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "line {} column {}", self.row, self.column)
+    }
+}
+
+impl Location {
+    pub fn visualize<'a>(
+        &self,
+        line: &'a str,
+        desc: impl fmt::Display + 'a,
+    ) -> impl fmt::Display + 'a {
+        struct Visualize<'a, D: fmt::Display> {
+            loc: Location,
+            line: &'a str,
+            desc: D,
+        }
+        impl<D: fmt::Display> fmt::Display for Visualize<'_, D> {
+            fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+                write!(
+                    f,
+                    "{}\n{}{arrow:>pad$}",
+                    self.desc,
+                    self.line,
+                    pad = self.loc.column,
+                    arrow = "^",
+                )
+            }
+        }
+        Visualize {
+            loc: *self,
+            line,
+            desc,
+        }
+    }
+}
+
+impl Location {
+    pub fn new(row: usize, column: usize) -> Self {
+        Location { row, column }
+    }
+
+    pub fn row(&self) -> usize {
+        self.row
+    }
+
+    pub fn column(&self) -> usize {
+        self.column
+    }
+
+    pub fn reset(&mut self) {
+        self.row = 1;
+        self.column = 1;
+    }
+
+    pub fn go_right(&mut self) {
+        self.column += 1;
+    }
+
+    pub fn go_left(&mut self) {
+        self.column -= 1;
+    }
+
+    pub fn newline(&mut self) {
+        self.row += 1;
+        self.column = 1;
+    }
+}
diff --git a/ast/src/unparse.rs b/ast/src/unparse.rs
new file mode 100644
index 00000000..70addae9
--- /dev/null
+++ b/ast/src/unparse.rs
@@ -0,0 +1,530 @@
+use crate::{
+    Arg, Arguments, Boolop, Cmpop, Comprehension, Constant, ConversionFlag, Expr, ExprKind,
+    Operator,
+};
+use std::fmt;
+
+mod precedence {
+    macro_rules! precedence {
+        ($($op:ident,)*) => {
+            precedence!(@0, $($op,)*);
+        };
+        (@$i:expr, $op1:ident, $($op:ident,)*) => {
+            pub const $op1: u8 = $i;
+            precedence!(@$i + 1, $($op,)*);
+        };
+        (@$i:expr,) => {};
+    }
+    precedence!(
+        TUPLE, TEST, OR, AND, NOT, CMP, // "EXPR" =
+        BOR, BXOR, BAND, SHIFT, ARITH, TERM, FACTOR, POWER, AWAIT, ATOM,
+    );
+    pub const EXPR: u8 = BOR;
+}
+
+#[repr(transparent)]
+struct Unparser<'a> {
+    f: fmt::Formatter<'a>,
+}
+impl<'a> Unparser<'a> {
+    fn new<'b>(f: &'b mut fmt::Formatter<'a>) -> &'b mut Unparser<'a> {
+        unsafe { &mut *(f as *mut fmt::Formatter<'a> as *mut Unparser<'a>) }
+    }
+
+    fn p(&mut self, s: &str) -> fmt::Result {
+        self.f.write_str(s)
+    }
+    fn p_if(&mut self, cond: bool, s: &str) -> fmt::Result {
+        if cond {
+            self.f.write_str(s)?;
+        }
+        Ok(())
+    }
+    fn p_delim(&mut self, first: &mut bool, s: &str) -> fmt::Result {
+        self.p_if(!std::mem::take(first), s)
+    }
+    fn write_fmt(&mut self, f: fmt::Arguments<'_>) -> fmt::Result {
+        self.f.write_fmt(f)
+    }
+
+    fn unparse_expr<U>(&mut self, ast: &Expr<U>, level: u8) -> fmt::Result {
+        macro_rules! opprec {
+            ($opty:ident, $x:expr, $enu:path, $($var:ident($op:literal, $prec:ident)),*$(,)?) => {
+                match $x {
+                    $(<$enu>::$var => (opprec!(@space $opty, $op), precedence::$prec),)*
+                }
+            };
+            (@space bin, $op:literal) => {
+                concat!(" ", $op, " ")
+            };
+            (@space un, $op:literal) => {
+                $op
+            };
+        }
+        macro_rules! group_if {
+            ($lvl:expr, $body:block) => {{
+                let group = level > $lvl;
+                self.p_if(group, "(")?;
+                let ret = $body;
+                self.p_if(group, ")")?;
+                ret
+            }};
+        }
+        match &ast.node {
+            ExprKind::BoolOp { op, values } => {
+                let (op, prec) = opprec!(bin, op, Boolop, And("and", AND), Or("or", OR));
+                group_if!(prec, {
+                    let mut first = true;
+                    for val in values {
+                        self.p_delim(&mut first, op)?;
+                        self.unparse_expr(val, prec + 1)?;
+                    }
+                })
+            }
+            ExprKind::NamedExpr { target, value } => {
+                group_if!(precedence::TUPLE, {
+                    self.unparse_expr(target, precedence::ATOM)?;
+                    self.p(" := ")?;
+                    self.unparse_expr(value, precedence::ATOM)?;
+                })
+            }
+            ExprKind::BinOp { left, op, right } => {
+                let rassoc = matches!(op, Operator::Pow);
+                let (op, prec) = opprec!(
+                    bin,
+                    op,
+                    Operator,
+                    Add("+", ARITH),
+                    Sub("-", ARITH),
+                    Mult("*", TERM),
+                    MatMult("@", TERM),
+                    Div("/", TERM),
+                    Mod("%", TERM),
+                    Pow("**", POWER),
+                    LShift("<<", SHIFT),
+                    RShift(">>", SHIFT),
+                    BitOr("|", BOR),
+                    BitXor("^", BXOR),
+                    BitAnd("&", BAND),
+                    FloorDiv("//", TERM),
+                );
+                group_if!(prec, {
+                    self.unparse_expr(left, prec + rassoc as u8)?;
+                    self.p(op)?;
+                    self.unparse_expr(right, prec + !rassoc as u8)?;
+                })
+            }
+            ExprKind::UnaryOp { op, operand } => {
+                let (op, prec) = opprec!(
+                    un,
+                    op,
+                    crate::Unaryop,
+                    Invert("~", FACTOR),
+                    Not("not ", NOT),
+                    UAdd("+", FACTOR),
+                    USub("-", FACTOR)
+                );
+                group_if!(prec, {
+                    self.p(op)?;
+                    self.unparse_expr(operand, prec)?;
+                })
+            }
+            ExprKind::Lambda { args, body } => {
+                group_if!(precedence::TEST, {
+                    let npos = args.args.len() + args.posonlyargs.len();
+                    self.p(if npos > 0 { "lambda " } else { "lambda" })?;
+                    self.unparse_args(args)?;
+                    write!(self, ": {}", **body)?;
+                })
+            }
+            ExprKind::IfExp { test, body, orelse } => {
+                group_if!(precedence::TEST, {
+                    self.unparse_expr(body, precedence::TEST + 1)?;
+                    self.p(" if ")?;
+                    self.unparse_expr(test, precedence::TEST + 1)?;
+                    self.p(" else ")?;
+                    self.unparse_expr(orelse, precedence::TEST)?;
+                })
+            }
+            ExprKind::Dict { keys, values } => {
+                self.p("{")?;
+                let mut first = true;
+                let (packed, unpacked) = values.split_at(keys.len());
+                for (k, v) in keys.iter().zip(packed) {
+                    self.p_delim(&mut first, ", ")?;
+                    write!(self, "{}: {}", *k, *v)?;
+                }
+                for d in unpacked {
+                    self.p_delim(&mut first, ", ")?;
+                    write!(self, "**{}", *d)?;
+                }
+                self.p("}")?;
+            }
+            ExprKind::Set { elts } => {
+                self.p("{")?;
+                let mut first = true;
+                for v in elts {
+                    self.p_delim(&mut first, ", ")?;
+                    self.unparse_expr(v, precedence::TEST)?;
+                }
+                self.p("}")?;
+            }
+            ExprKind::ListComp { elt, generators } => {
+                self.p("[")?;
+                self.unparse_expr(elt, precedence::TEST)?;
+                self.unparse_comp(generators)?;
+                self.p("]")?;
+            }
+            ExprKind::SetComp { elt, generators } => {
+                self.p("{")?;
+                self.unparse_expr(elt, precedence::TEST)?;
+                self.unparse_comp(generators)?;
+                self.p("}")?;
+            }
+            ExprKind::DictComp {
+                key,
+                value,
+                generators,
+            } => {
+                self.p("{")?;
+                self.unparse_expr(key, precedence::TEST)?;
+                self.p(": ")?;
+                self.unparse_expr(value, precedence::TEST)?;
+                self.unparse_comp(generators)?;
+                self.p("}")?;
+            }
+            ExprKind::GeneratorExp { elt, generators } => {
+                self.p("(")?;
+                self.unparse_expr(elt, precedence::TEST)?;
+                self.unparse_comp(generators)?;
+                self.p(")")?;
+            }
+            ExprKind::Await { value } => {
+                group_if!(precedence::AWAIT, {
+                    self.p("await ")?;
+                    self.unparse_expr(value, precedence::ATOM)?;
+                })
+            }
+            ExprKind::Yield { value } => {
+                if let Some(value) = value {
+                    write!(self, "(yield {})", **value)?;
+                } else {
+                    self.p("(yield)")?;
+                }
+            }
+            ExprKind::YieldFrom { value } => {
+                write!(self, "(yield from {})", **value)?;
+            }
+            ExprKind::Compare {
+                left,
+                ops,
+                comparators,
+            } => {
+                group_if!(precedence::CMP, {
+                    let new_lvl = precedence::CMP + 1;
+                    self.unparse_expr(left, new_lvl)?;
+                    for (op, cmp) in ops.iter().zip(comparators) {
+                        let op = match op {
+                            Cmpop::Eq => " == ",
+                            Cmpop::NotEq => " != ",
+                            Cmpop::Lt => " < ",
+                            Cmpop::LtE => " <= ",
+                            Cmpop::Gt => " > ",
+                            Cmpop::GtE => " >= ",
+                            Cmpop::Is => " is ",
+                            Cmpop::IsNot => " is not ",
+                            Cmpop::In => " in ",
+                            Cmpop::NotIn => " not in ",
+                        };
+                        self.p(op)?;
+                        self.unparse_expr(cmp, new_lvl)?;
+                    }
+                })
+            }
+            ExprKind::Call {
+                func,
+                args,
+                keywords,
+            } => {
+                self.unparse_expr(func, precedence::ATOM)?;
+                self.p("(")?;
+                if let (
+                    [Expr {
+                        node: ExprKind::GeneratorExp { elt, generators },
+                        ..
+                    }],
+                    [],
+                ) = (&**args, &**keywords)
+                {
+                    // make sure a single genexp doesn't get double parens
+                    self.unparse_expr(elt, precedence::TEST)?;
+                    self.unparse_comp(generators)?;
+                } else {
+                    let mut first = true;
+                    for arg in args {
+                        self.p_delim(&mut first, ", ")?;
+                        self.unparse_expr(arg, precedence::TEST)?;
+                    }
+                    for kw in keywords {
+                        self.p_delim(&mut first, ", ")?;
+                        if let Some(arg) = &kw.node.arg {
+                            self.p(arg)?;
+                            self.p("=")?;
+                        } else {
+                            self.p("**")?;
+                        }
+                        self.unparse_expr(&kw.node.value, precedence::TEST)?;
+                    }
+                }
+                self.p(")")?;
+            }
+            ExprKind::FormattedValue {
+                value,
+                conversion,
+                format_spec,
+            } => self.unparse_formatted(value, *conversion, format_spec.as_deref())?,
+            ExprKind::JoinedStr { values } => self.unparse_joinedstr(values, false)?,
+            ExprKind::Constant { value, kind } => {
+                if let Some(kind) = kind {
+                    self.p(kind)?;
+                }
+                assert_eq!(f64::MAX_10_EXP, 308);
+                let inf_str = "1e309";
+                match value {
+                    Constant::Float(f) if f.is_infinite() => self.p(inf_str)?,
+                    Constant::Complex { real, imag }
+                        if real.is_infinite() || imag.is_infinite() =>
+                    {
+                        self.p(&value.to_string().replace("inf", inf_str))?
+                    }
+                    _ => fmt::Display::fmt(value, &mut self.f)?,
+                }
+            }
+            ExprKind::Attribute { value, attr, .. } => {
+                self.unparse_expr(value, precedence::ATOM)?;
+                let period = if let ExprKind::Constant {
+                    value: Constant::Int(_),
+                    ..
+                } = &value.node
+                {
+                    " ."
+                } else {
+                    "."
+                };
+                self.p(period)?;
+                self.p(attr)?;
+            }
+            ExprKind::Subscript { value, slice, .. } => {
+                self.unparse_expr(value, precedence::ATOM)?;
+                let mut lvl = precedence::TUPLE;
+                if let ExprKind::Tuple { elts, .. } = &slice.node {
+                    if elts
+                        .iter()
+                        .any(|expr| matches!(expr.node, ExprKind::Starred { .. }))
+                    {
+                        lvl += 1
+                    }
+                }
+                self.p("[")?;
+                self.unparse_expr(slice, lvl)?;
+                self.p("]")?;
+            }
+            ExprKind::Starred { value, .. } => {
+                self.p("*")?;
+                self.unparse_expr(value, precedence::EXPR)?;
+            }
+            ExprKind::Name { id, .. } => self.p(id)?,
+            ExprKind::List { elts, .. } => {
+                self.p("[")?;
+                let mut first = true;
+                for elt in elts {
+                    self.p_delim(&mut first, ", ")?;
+                    self.unparse_expr(elt, precedence::TEST)?;
+                }
+                self.p("]")?;
+            }
+            ExprKind::Tuple { elts, .. } => {
+                if elts.is_empty() {
+                    self.p("()")?;
+                } else {
+                    group_if!(precedence::TUPLE, {
+                        let mut first = true;
+                        for elt in elts {
+                            self.p_delim(&mut first, ", ")?;
+                            self.unparse_expr(elt, precedence::TEST)?;
+                        }
+                        self.p_if(elts.len() == 1, ",")?;
+                    })
+                }
+            }
+            ExprKind::Slice { lower, upper, step } => {
+                if let Some(lower) = lower {
+                    self.unparse_expr(lower, precedence::TEST)?;
+                }
+                self.p(":")?;
+                if let Some(upper) = upper {
+                    self.unparse_expr(upper, precedence::TEST)?;
+                }
+                if let Some(step) = step {
+                    self.p(":")?;
+                    self.unparse_expr(step, precedence::TEST)?;
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn unparse_args<U>(&mut self, args: &Arguments<U>) -> fmt::Result {
+        let mut first = true;
+        let defaults_start = args.posonlyargs.len() + args.args.len() - args.defaults.len();
+        for (i, arg) in args.posonlyargs.iter().chain(&args.args).enumerate() {
+            self.p_delim(&mut first, ", ")?;
+            self.unparse_arg(arg)?;
+            if let Some(i) = i.checked_sub(defaults_start) {
+                write!(self, "={}", &args.defaults[i])?;
+            }
+            self.p_if(i + 1 == args.posonlyargs.len(), ", /")?;
+        }
+        if args.vararg.is_some() || !args.kwonlyargs.is_empty() {
+            self.p_delim(&mut first, ", ")?;
+            self.p("*")?;
+        }
+        if let Some(vararg) = &args.vararg {
+            self.unparse_arg(vararg)?;
+        }
+        let defaults_start = args.kwonlyargs.len() - args.kw_defaults.len();
+        for (i, kwarg) in args.kwonlyargs.iter().enumerate() {
+            self.p_delim(&mut first, ", ")?;
+            self.unparse_arg(kwarg)?;
+            if let Some(default) = i
+                .checked_sub(defaults_start)
+                .and_then(|i| args.kw_defaults.get(i))
+            {
+                write!(self, "={}", default)?;
+            }
+        }
+        if let Some(kwarg) = &args.kwarg {
+            self.p_delim(&mut first, ", ")?;
+            self.p("**")?;
+            self.unparse_arg(kwarg)?;
+        }
+        Ok(())
+    }
+    fn unparse_arg<U>(&mut self, arg: &Arg<U>) -> fmt::Result {
+        self.p(&arg.node.arg)?;
+        if let Some(ann) = &arg.node.annotation {
+            write!(self, ": {}", **ann)?;
+        }
+        Ok(())
+    }
+
+    fn unparse_comp<U>(&mut self, generators: &[Comprehension<U>]) -> fmt::Result {
+        for comp in generators {
+            self.p(if comp.is_async > 0 {
+                " async for "
+            } else {
+                " for "
+            })?;
+            self.unparse_expr(&comp.target, precedence::TUPLE)?;
+            self.p(" in ")?;
+            self.unparse_expr(&comp.iter, precedence::TEST + 1)?;
+            for cond in &comp.ifs {
+                self.p(" if ")?;
+                self.unparse_expr(cond, precedence::TEST + 1)?;
+            }
+        }
+        Ok(())
+    }
+
+    fn unparse_fstring_body<U>(&mut self, values: &[Expr<U>], is_spec: bool) -> fmt::Result {
+        for value in values {
+            self.unparse_fstring_elem(value, is_spec)?;
+        }
+        Ok(())
+    }
+
+    fn unparse_formatted<U>(
+        &mut self,
+        val: &Expr<U>,
+        conversion: usize,
+        spec: Option<&Expr<U>>,
+    ) -> fmt::Result {
+        let buffered = to_string_fmt(|f| Unparser::new(f).unparse_expr(val, precedence::TEST + 1));
+        let brace = if buffered.starts_with('{') {
+            // put a space to avoid escaping the bracket
+            "{ "
+        } else {
+            "{"
+        };
+        self.p(brace)?;
+        self.p(&buffered)?;
+        drop(buffered);
+
+        if conversion != ConversionFlag::None as usize {
+            self.p("!")?;
+            let buf = &[conversion as u8];
+            let c = std::str::from_utf8(buf).unwrap();
+            self.p(c)?;
+        }
+
+        if let Some(spec) = spec {
+            self.p(":")?;
+            self.unparse_fstring_elem(spec, true)?;
+        }
+
+        self.p("}")?;
+
+        Ok(())
+    }
+
+    fn unparse_fstring_elem<U>(&mut self, expr: &Expr<U>, is_spec: bool) -> fmt::Result {
+        match &expr.node {
+            ExprKind::Constant { value, .. } => {
+                if let Constant::Str(s) = value {
+                    self.unparse_fstring_str(s)
+                } else {
+                    unreachable!()
+                }
+            }
+            ExprKind::JoinedStr { values } => self.unparse_joinedstr(values, is_spec),
+            ExprKind::FormattedValue {
+                value,
+                conversion,
+                format_spec,
+            } => self.unparse_formatted(value, *conversion, format_spec.as_deref()),
+            _ => unreachable!(),
+        }
+    }
+
+    fn unparse_fstring_str(&mut self, s: &str) -> fmt::Result {
+        let s = s.replace('{', "{{").replace('}', "}}");
+        self.p(&s)
+    }
+
+    fn unparse_joinedstr<U>(&mut self, values: &[Expr<U>], is_spec: bool) -> fmt::Result {
+        if is_spec {
+            self.unparse_fstring_body(values, is_spec)
+        } else {
+            self.p("f")?;
+            let body = to_string_fmt(|f| Unparser::new(f).unparse_fstring_body(values, is_spec));
+            fmt::Display::fmt(&rustpython_common::str::repr(&body), &mut self.f)
+        }
+    }
+}
+
+impl<U> fmt::Display for Expr<U> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        Unparser::new(f).unparse_expr(self, precedence::TEST)
+    }
+}
+
+fn to_string_fmt(f: impl FnOnce(&mut fmt::Formatter) -> fmt::Result) -> String {
+    use std::cell::Cell;
+    struct Fmt<F>(Cell<Option<F>>);
+    impl<F: FnOnce(&mut fmt::Formatter) -> fmt::Result> fmt::Display for Fmt<F> {
+        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+            self.0.take().unwrap()(f)
+        }
+    }
+    Fmt(Cell::new(Some(f))).to_string()
+}
diff --git a/bytecode/Cargo.toml b/bytecode/Cargo.toml
new file mode 100644
index 00000000..1457072b
--- /dev/null
+++ b/bytecode/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "rustpython-bytecode"
+description = "RustPython specific bytecode."
+version = "0.1.2"
+authors = ["RustPython Team"]
+edition = "2021"
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
+
+
+[dependencies]
+bincode = "1.3.3"
+bitflags = "1.3.2"
+lz4_flex = "0.9.2"
+num-bigint = { version = "0.4.3", features = ["serde"] }
+num-complex = { version = "0.4.0", features = ["serde"] }
+serde = { version = "1.0.136", features = ["derive"] }
+itertools = "0.10.3"
+bstr = "0.2.17"
+static_assertions = "1.1.0"
diff --git a/bytecode/src/lib.rs b/bytecode/src/lib.rs
new file mode 100644
index 00000000..f4f3f306
--- /dev/null
+++ b/bytecode/src/lib.rs
@@ -0,0 +1,1395 @@
+//! Implement python as a virtual machine with bytecodes. This module
+//! implements bytecode structure.
+
+#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
+#![doc(html_root_url = "https://docs.rs/rustpython-bytecode/")]
+
+use bitflags::bitflags;
+use bstr::ByteSlice;
+use itertools::Itertools;
+use num_bigint::BigInt;
+use num_complex::Complex64;
+use serde::{Deserialize, Serialize};
+use std::{collections::BTreeSet, fmt, hash};
+
+/// Sourcecode location.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Location {
+    row: u32,
+    column: u32,
+}
+
+impl Location {
+    /// Creates a new Location object at the given row and column.
+    ///
+    /// # Example
+    /// ```
+    /// use rustpython_bytecode::Location;
+    /// let loc = Location::new(10, 10);
+    /// ```
+    pub fn new(row: usize, column: usize) -> Self {
+        let row = row.try_into().expect("Location::row over u32");
+        let column = column.try_into().expect("Location::column over u32");
+        Location { row, column }
+    }
+
+    /// Current row
+    pub fn row(&self) -> usize {
+        self.row as usize
+    }
+
+    /// Current column
+    pub fn column(&self) -> usize {
+        self.column as usize
+    }
+}
+
+pub trait Constant: Sized {
+    type Name: AsRef<str>;
+
+    /// Transforms the given Constant to a BorrowedConstant
+    fn borrow_constant(&self) -> BorrowedConstant<Self>;
+}
+
+impl Constant for ConstantData {
+    type Name = String;
+    fn borrow_constant(&self) -> BorrowedConstant<Self> {
+        use BorrowedConstant::*;
+        match self {
+            ConstantData::Integer { value } => Integer { value },
+            ConstantData::Float { value } => Float { value: *value },
+            ConstantData::Complex { value } => Complex { value: *value },
+            ConstantData::Boolean { value } => Boolean { value: *value },
+            ConstantData::Str { value } => Str { value },
+            ConstantData::Bytes { value } => Bytes { value },
+            ConstantData::Code { code } => Code { code },
+            ConstantData::Tuple { elements } => Tuple {
+                elements: Box::new(elements.iter().map(|e| e.borrow_constant())),
+            },
+            ConstantData::None => None,
+            ConstantData::Ellipsis => Ellipsis,
+        }
+    }
+}
+
+/// A Constant Bag
+pub trait ConstantBag: Sized + Copy {
+    type Constant: Constant;
+    fn make_constant<C: Constant>(&self, constant: BorrowedConstant<C>) -> Self::Constant;
+    fn make_name(&self, name: &str) -> <Self::Constant as Constant>::Name;
+}
+
+#[derive(Clone, Copy)]
+pub struct BasicBag;
+
+impl ConstantBag for BasicBag {
+    type Constant = ConstantData;
+    fn make_constant<C: Constant>(&self, constant: BorrowedConstant<C>) -> Self::Constant {
+        constant.to_owned()
+    }
+    fn make_name(&self, name: &str) -> <Self::Constant as Constant>::Name {
+        name.to_owned()
+    }
+}
+
+/// Primary container of a single code object. Each python function has
+/// a codeobject. Also a module has a codeobject.
+#[derive(Clone, Serialize, Deserialize)]
+pub struct CodeObject<C: Constant = ConstantData> {
+    pub instructions: Box<[Instruction]>,
+    pub locations: Box<[Location]>,
+    pub flags: CodeFlags,
+    pub posonlyarg_count: usize,
+    // Number of positional-only arguments
+    pub arg_count: usize,
+    pub kwonlyarg_count: usize,
+    pub source_path: C::Name,
+    pub first_line_number: usize,
+    pub max_stackdepth: u32,
+    pub obj_name: C::Name,
+    // Name of the object that created this code object
+    pub cell2arg: Option<Box<[isize]>>,
+    pub constants: Box<[C]>,
+    #[serde(bound(
+        deserialize = "C::Name: serde::Deserialize<'de>",
+        serialize = "C::Name: serde::Serialize"
+    ))]
+    pub names: Box<[C::Name]>,
+    pub varnames: Box<[C::Name]>,
+    pub cellvars: Box<[C::Name]>,
+    pub freevars: Box<[C::Name]>,
+}
+
+bitflags! {
+    #[derive(Serialize, Deserialize)]
+    pub struct CodeFlags: u16 {
+        const NEW_LOCALS = 0x01;
+        const IS_GENERATOR = 0x02;
+        const IS_COROUTINE = 0x04;
+        const HAS_VARARGS = 0x08;
+        const HAS_VARKEYWORDS = 0x10;
+        const IS_OPTIMIZED = 0x20;
+    }
+}
+
+impl CodeFlags {
+    pub const NAME_MAPPING: &'static [(&'static str, CodeFlags)] = &[
+        ("GENERATOR", CodeFlags::IS_GENERATOR),
+        ("COROUTINE", CodeFlags::IS_COROUTINE),
+        (
+            "ASYNC_GENERATOR",
+            Self::from_bits_truncate(Self::IS_GENERATOR.bits | Self::IS_COROUTINE.bits),
+        ),
+        ("VARARGS", CodeFlags::HAS_VARARGS),
+        ("VARKEYWORDS", CodeFlags::HAS_VARKEYWORDS),
+    ];
+}
+
+#[derive(Serialize, Debug, Deserialize, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)]
+#[repr(transparent)]
+// XXX: if you add a new instruction that stores a Label, make sure to add it in
+// Instruction::label_arg{,_mut}
+pub struct Label(pub u32);
+
+impl fmt::Display for Label {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+/// Transforms a value prior to formatting it.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[repr(u8)]
+pub enum ConversionFlag {
+    /// No conversion
+    None = 0, // CPython uses -1 but not pleasure for us
+    /// Converts by calling `str(<value>)`.
+    Str = b's',
+    /// Converts by calling `ascii(<value>)`.
+    Ascii = b'a',
+    /// Converts by calling `repr(<value>)`.
+    Repr = b'r',
+}
+
+impl TryFrom<usize> for ConversionFlag {
+    type Error = usize;
+    fn try_from(b: usize) -> Result<Self, Self::Error> {
+        let b = b.try_into().map_err(|_| b)?;
+        match b {
+            0 => Ok(Self::None),
+            b's' => Ok(Self::Str),
+            b'a' => Ok(Self::Ascii),
+            b'r' => Ok(Self::Repr),
+            b => Err(b as usize),
+        }
+    }
+}
+
+/// The kind of Raise that occurred.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum RaiseKind {
+    Reraise,
+    Raise,
+    RaiseCause,
+}
+
+pub type NameIdx = u32;
+
+/// A Single bytecode instruction.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Instruction {
+    /// Importing by name
+    ImportName {
+        idx: NameIdx,
+    },
+    /// Importing without name
+    ImportNameless,
+    /// Import *
+    ImportStar,
+    /// from ... import ...
+    ImportFrom {
+        idx: NameIdx,
+    },
+    LoadFast(NameIdx),
+    LoadNameAny(NameIdx),
+    LoadGlobal(NameIdx),
+    LoadDeref(NameIdx),
+    LoadClassDeref(NameIdx),
+    StoreFast(NameIdx),
+    StoreLocal(NameIdx),
+    StoreGlobal(NameIdx),
+    StoreDeref(NameIdx),
+    DeleteFast(NameIdx),
+    DeleteLocal(NameIdx),
+    DeleteGlobal(NameIdx),
+    DeleteDeref(NameIdx),
+    LoadClosure(NameIdx),
+    Subscript,
+    StoreSubscript,
+    DeleteSubscript,
+    StoreAttr {
+        idx: NameIdx,
+    },
+    DeleteAttr {
+        idx: NameIdx,
+    },
+    LoadConst {
+        /// index into constants vec
+        idx: u32,
+    },
+    UnaryOperation {
+        op: UnaryOperator,
+    },
+    BinaryOperation {
+        op: BinaryOperator,
+    },
+    BinaryOperationInplace {
+        op: BinaryOperator,
+    },
+    LoadAttr {
+        idx: NameIdx,
+    },
+    TestOperation {
+        op: TestOperator,
+    },
+    CompareOperation {
+        op: ComparisonOperator,
+    },
+    Pop,
+    Rotate2,
+    Rotate3,
+    Duplicate,
+    Duplicate2,
+    GetIter,
+    Continue {
+        target: Label,
+    },
+    Break {
+        target: Label,
+    },
+    Jump {
+        target: Label,
+    },
+    /// Pop the top of the stack, and jump if this value is true.
+    JumpIfTrue {
+        target: Label,
+    },
+    /// Pop the top of the stack, and jump if this value is false.
+    JumpIfFalse {
+        target: Label,
+    },
+    /// Peek at the top of the stack, and jump if this value is true.
+    /// Otherwise, pop top of stack.
+    JumpIfTrueOrPop {
+        target: Label,
+    },
+    /// Peek at the top of the stack, and jump if this value is false.
+    /// Otherwise, pop top of stack.
+    JumpIfFalseOrPop {
+        target: Label,
+    },
+    MakeFunction(MakeFunctionFlags),
+    CallFunctionPositional {
+        nargs: u32,
+    },
+    CallFunctionKeyword {
+        nargs: u32,
+    },
+    CallFunctionEx {
+        has_kwargs: bool,
+    },
+    LoadMethod {
+        idx: NameIdx,
+    },
+    CallMethodPositional {
+        nargs: u32,
+    },
+    CallMethodKeyword {
+        nargs: u32,
+    },
+    CallMethodEx {
+        has_kwargs: bool,
+    },
+    ForIter {
+        target: Label,
+    },
+    ReturnValue,
+    YieldValue,
+    YieldFrom,
+    SetupAnnotation,
+    SetupLoop {
+        break_target: Label,
+    },
+
+    /// Setup a finally handler, which will be called whenever one of this events occurs:
+    /// - the block is popped
+    /// - the function returns
+    /// - an exception is returned
+    SetupFinally {
+        handler: Label,
+    },
+
+    /// Enter a finally block, without returning, excepting, just because we are there.
+    EnterFinally,
+
+    /// Marker bytecode for the end of a finally sequence.
+    /// When this bytecode is executed, the eval loop does one of those things:
+    /// - Continue at a certain bytecode position
+    /// - Propagate the exception
+    /// - Return from a function
+    /// - Do nothing at all, just continue
+    EndFinally,
+
+    SetupExcept {
+        handler: Label,
+    },
+    SetupWith {
+        end: Label,
+    },
+    WithCleanupStart,
+    WithCleanupFinish,
+    PopBlock,
+    Raise {
+        kind: RaiseKind,
+    },
+    BuildString {
+        size: u32,
+    },
+    BuildTuple {
+        unpack: bool,
+        size: u32,
+    },
+    BuildList {
+        unpack: bool,
+        size: u32,
+    },
+    BuildSet {
+        unpack: bool,
+        size: u32,
+    },
+    BuildMap {
+        unpack: bool,
+        for_call: bool,
+        size: u32,
+    },
+    DictUpdate,
+    BuildSlice {
+        /// whether build a slice with a third step argument
+        step: bool,
+    },
+    ListAppend {
+        i: u32,
+    },
+    SetAdd {
+        i: u32,
+    },
+    MapAdd {
+        i: u32,
+    },
+
+    PrintExpr,
+    LoadBuildClass,
+    UnpackSequence {
+        size: u32,
+    },
+    UnpackEx {
+        before: u8,
+        after: u8,
+    },
+    FormatValue {
+        conversion: ConversionFlag,
+    },
+    PopException,
+    Reverse {
+        amount: u32,
+    },
+    GetAwaitable,
+    BeforeAsyncWith,
+    SetupAsyncWith {
+        end: Label,
+    },
+    GetAIter,
+    GetANext,
+    EndAsyncFor,
+}
+static_assertions::assert_eq_size!(Instruction, u64);
+
+use self::Instruction::*;
+
+bitflags! {
+    #[derive(Serialize, Deserialize)]
+    pub struct MakeFunctionFlags: u8 {
+        const CLOSURE = 0x01;
+        const ANNOTATIONS = 0x02;
+        const KW_ONLY_DEFAULTS = 0x04;
+        const DEFAULTS = 0x08;
+    }
+}
+
+/// A Constant (which usually encapsulates data within it)
+///
+/// # Examples
+/// ```
+/// use rustpython_bytecode::ConstantData;
+/// let a = ConstantData::Float {value: 120f64};
+/// let b = ConstantData::Boolean {value: false};
+/// assert_ne!(a, b);
+/// ```
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum ConstantData {
+    Tuple { elements: Vec<ConstantData> },
+    Integer { value: BigInt },
+    Float { value: f64 },
+    Complex { value: Complex64 },
+    Boolean { value: bool },
+    Str { value: String },
+    Bytes { value: Vec<u8> },
+    Code { code: Box<CodeObject> },
+    None,
+    Ellipsis,
+}
+
+impl PartialEq for ConstantData {
+    fn eq(&self, other: &Self) -> bool {
+        use ConstantData::*;
+        match (self, other) {
+            (Integer { value: a }, Integer { value: b }) => a == b,
+            // we want to compare floats *by actual value* - if we have the *exact same* float
+            // already in a constant cache, we want to use that
+            (Float { value: a }, Float { value: b }) => a.to_bits() == b.to_bits(),
+            (Complex { value: a }, Complex { value: b }) => {
+                a.re.to_bits() == b.re.to_bits() && a.im.to_bits() == b.im.to_bits()
+            }
+            (Boolean { value: a }, Boolean { value: b }) => a == b,
+            (Str { value: a }, Str { value: b }) => a == b,
+            (Bytes { value: a }, Bytes { value: b }) => a == b,
+            (Code { code: a }, Code { code: b }) => std::ptr::eq(a.as_ref(), b.as_ref()),
+            (Tuple { elements: a }, Tuple { elements: b }) => a == b,
+            (None, None) => true,
+            (Ellipsis, Ellipsis) => true,
+            _ => false,
+        }
+    }
+}
+
+impl Eq for ConstantData {}
+
+impl hash::Hash for ConstantData {
+    fn hash<H: hash::Hasher>(&self, state: &mut H) {
+        use ConstantData::*;
+        std::mem::discriminant(self).hash(state);
+        match self {
+            Integer { value } => value.hash(state),
+            Float { value } => value.to_bits().hash(state),
+            Complex { value } => {
+                value.re.to_bits().hash(state);
+                value.im.to_bits().hash(state);
+            }
+            Boolean { value } => value.hash(state),
+            Str { value } => value.hash(state),
+            Bytes { value } => value.hash(state),
+            Code { code } => std::ptr::hash(code.as_ref(), state),
+            Tuple { elements } => elements.hash(state),
+            None => {}
+            Ellipsis => {}
+        }
+    }
+}
+
+/// A borrowed Constant
+pub enum BorrowedConstant<'a, C: Constant> {
+    Integer { value: &'a BigInt },
+    Float { value: f64 },
+    Complex { value: Complex64 },
+    Boolean { value: bool },
+    Str { value: &'a str },
+    Bytes { value: &'a [u8] },
+    Code { code: &'a CodeObject<C> },
+    Tuple { elements: BorrowedTupleIter<'a, C> },
+    None,
+    Ellipsis,
+}
+
+type BorrowedTupleIter<'a, C> = Box<dyn Iterator<Item = BorrowedConstant<'a, C>> + 'a>;
+
+impl<C: Constant> BorrowedConstant<'_, C> {
+    // takes `self` because we need to consume the iterator
+    pub fn fmt_display(self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            BorrowedConstant::Integer { value } => write!(f, "{}", value),
+            BorrowedConstant::Float { value } => write!(f, "{}", value),
+            BorrowedConstant::Complex { value } => write!(f, "{}", value),
+            BorrowedConstant::Boolean { value } => {
+                write!(f, "{}", if value { "True" } else { "False" })
+            }
+            BorrowedConstant::Str { value } => write!(f, "{:?}", value),
+            BorrowedConstant::Bytes { value } => write!(f, "b{:?}", value.as_bstr()),
+            BorrowedConstant::Code { code } => write!(f, "{:?}", code),
+            BorrowedConstant::Tuple { elements } => {
+                write!(f, "(")?;
+                let mut first = true;
+                for c in elements {
+                    if first {
+                        first = false
+                    } else {
+                        write!(f, ", ")?;
+                    }
+                    c.fmt_display(f)?;
+                }
+                write!(f, ")")
+            }
+            BorrowedConstant::None => write!(f, "None"),
+            BorrowedConstant::Ellipsis => write!(f, "..."),
+        }
+    }
+    pub fn to_owned(self) -> ConstantData {
+        use ConstantData::*;
+        match self {
+            BorrowedConstant::Integer { value } => Integer {
+                value: value.clone(),
+            },
+            BorrowedConstant::Float { value } => Float { value },
+            BorrowedConstant::Complex { value } => Complex { value },
+            BorrowedConstant::Boolean { value } => Boolean { value },
+            BorrowedConstant::Str { value } => Str {
+                value: value.to_owned(),
+            },
+            BorrowedConstant::Bytes { value } => Bytes {
+                value: value.to_owned(),
+            },
+            BorrowedConstant::Code { code } => Code {
+                code: Box::new(code.map_clone_bag(&BasicBag)),
+            },
+            BorrowedConstant::Tuple { elements } => Tuple {
+                elements: elements.map(BorrowedConstant::to_owned).collect(),
+            },
+            BorrowedConstant::None => None,
+            BorrowedConstant::Ellipsis => Ellipsis,
+        }
+    }
+}
+
+/// The possible comparison operators
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum ComparisonOperator {
+    // be intentional with bits so that we can do eval_ord with just a bitwise and
+    // bits: | Equal | Greater | Less |
+    Less = 0b001,
+    Greater = 0b010,
+    NotEqual = 0b011,
+    Equal = 0b100,
+    LessOrEqual = 0b101,
+    GreaterOrEqual = 0b110,
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum TestOperator {
+    In,
+    NotIn,
+    Is,
+    IsNot,
+    /// two exceptions that match?
+    ExceptionMatch,
+}
+
+/// The possible Binary operators
+/// # Examples
+///
+/// ```
+/// use rustpython_bytecode::Instruction::BinaryOperation;
+/// use rustpython_bytecode::BinaryOperator::Add;
+/// let op = BinaryOperation {op: Add};
+/// ```
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum BinaryOperator {
+    Power,
+    Multiply,
+    MatrixMultiply,
+    Divide,
+    FloorDivide,
+    Modulo,
+    Add,
+    Subtract,
+    Lshift,
+    Rshift,
+    And,
+    Xor,
+    Or,
+}
+
+/// The possible unary operators
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum UnaryOperator {
+    Not,
+    Invert,
+    Minus,
+    Plus,
+}
+
+/*
+Maintain a stack of blocks on the VM.
+pub enum BlockType {
+    Loop,
+    Except,
+}
+*/
+
+/// Argument structure
+pub struct Arguments<'a, N: AsRef<str>> {
+    pub posonlyargs: &'a [N],
+    pub args: &'a [N],
+    pub vararg: Option<&'a N>,
+    pub kwonlyargs: &'a [N],
+    pub varkwarg: Option<&'a N>,
+}
+
+impl<N: AsRef<str>> fmt::Debug for Arguments<'_, N> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        macro_rules! fmt_slice {
+            ($x:expr) => {
+                format_args!("[{}]", $x.iter().map(AsRef::as_ref).format(", "))
+            };
+        }
+        f.debug_struct("Arguments")
+            .field("posonlyargs", &fmt_slice!(self.posonlyargs))
+            .field("args", &fmt_slice!(self.posonlyargs))
+            .field("vararg", &self.vararg.map(N::as_ref))
+            .field("kwonlyargs", &fmt_slice!(self.kwonlyargs))
+            .field("varkwarg", &self.varkwarg.map(N::as_ref))
+            .finish()
+    }
+}
+
+impl<C: Constant> CodeObject<C> {
+    /// Get all arguments of the code object
+    /// like inspect.getargs
+    pub fn arg_names(&self) -> Arguments<C::Name> {
+        let nargs = self.arg_count;
+        let nkwargs = self.kwonlyarg_count;
+        let mut varargspos = nargs + nkwargs;
+        let posonlyargs = &self.varnames[..self.posonlyarg_count];
+        let args = &self.varnames[..nargs];
+        let kwonlyargs = &self.varnames[nargs..varargspos];
+
+        let vararg = if self.flags.contains(CodeFlags::HAS_VARARGS) {
+            let vararg = &self.varnames[varargspos];
+            varargspos += 1;
+            Some(vararg)
+        } else {
+            None
+        };
+        let varkwarg = if self.flags.contains(CodeFlags::HAS_VARKEYWORDS) {
+            Some(&self.varnames[varargspos])
+        } else {
+            None
+        };
+
+        Arguments {
+            posonlyargs,
+            args,
+            vararg,
+            kwonlyargs,
+            varkwarg,
+        }
+    }
+
+    /// Return the labels targeted by the instructions of this CodeObject
+    pub fn label_targets(&self) -> BTreeSet<Label> {
+        let mut label_targets = BTreeSet::new();
+        for instruction in &*self.instructions {
+            if let Some(l) = instruction.label_arg() {
+                label_targets.insert(*l);
+            }
+        }
+        label_targets
+    }
+
+    fn display_inner(
+        &self,
+        f: &mut fmt::Formatter,
+        expand_codeobjects: bool,
+        level: usize,
+    ) -> fmt::Result {
+        let label_targets = self.label_targets();
+        let line_digits = (3).max(self.locations.last().unwrap().row.to_string().len());
+        let offset_digits = (4).max(self.instructions.len().to_string().len());
+        let mut last_line = u32::MAX;
+        for (offset, instruction) in self.instructions.iter().enumerate() {
+            // optional line number
+            let line = self.locations[offset].row;
+            if line != last_line {
+                if last_line != u32::MAX {
+                    writeln!(f)?;
+                }
+                last_line = line;
+                write!(f, "{line:line_digits$}")?;
+            } else {
+                for _ in 0..line_digits {
+                    write!(f, " ")?;
+                }
+            }
+            write!(f, " ")?;
+
+            // level indent
+            for _ in 0..level {
+                write!(f, "    ")?;
+            }
+
+            // arrow and offset
+            let arrow = if label_targets.contains(&Label(offset as u32)) {
+                ">>"
+            } else {
+                "  "
+            };
+            write!(f, "{arrow} {offset:offset_digits$} ")?;
+
+            // instruction
+            instruction.fmt_dis(
+                f,
+                &self.constants,
+                &self.names,
+                &self.varnames,
+                &self.cellvars,
+                &self.freevars,
+                expand_codeobjects,
+                level,
+            )?;
+        }
+        Ok(())
+    }
+
+    /// Recursively display this CodeObject
+    pub fn display_expand_codeobjects(&self) -> impl fmt::Display + '_ {
+        struct Display<'a, C: Constant>(&'a CodeObject<C>);
+        impl<C: Constant> fmt::Display for Display<'_, C> {
+            fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+                self.0.display_inner(f, true, 1)
+            }
+        }
+        Display(self)
+    }
+
+    /// Map this CodeObject to one that holds a Bag::Constant
+    pub fn map_bag<Bag: ConstantBag>(self, bag: Bag) -> CodeObject<Bag::Constant> {
+        let map_names = |names: Box<[C::Name]>| {
+            names
+                .into_vec()
+                .into_iter()
+                .map(|x| bag.make_name(x.as_ref()))
+                .collect::<Box<[_]>>()
+        };
+        CodeObject {
+            constants: self
+                .constants
+                .into_vec()
+                .into_iter()
+                .map(|x| bag.make_constant(x.borrow_constant()))
+                .collect(),
+            names: map_names(self.names),
+            varnames: map_names(self.varnames),
+            cellvars: map_names(self.cellvars),
+            freevars: map_names(self.freevars),
+            source_path: bag.make_name(self.source_path.as_ref()),
+            obj_name: bag.make_name(self.obj_name.as_ref()),
+
+            instructions: self.instructions,
+            locations: self.locations,
+            flags: self.flags,
+            posonlyarg_count: self.posonlyarg_count,
+            arg_count: self.arg_count,
+            kwonlyarg_count: self.kwonlyarg_count,
+            first_line_number: self.first_line_number,
+            max_stackdepth: self.max_stackdepth,
+            cell2arg: self.cell2arg,
+        }
+    }
+
+    /// Same as `map_bag` but clones `self`
+    pub fn map_clone_bag<Bag: ConstantBag>(&self, bag: &Bag) -> CodeObject<Bag::Constant> {
+        let map_names =
+            |names: &[C::Name]| names.iter().map(|x| bag.make_name(x.as_ref())).collect();
+        CodeObject {
+            constants: self
+                .constants
+                .iter()
+                .map(|x| bag.make_constant(x.borrow_constant()))
+                .collect(),
+            names: map_names(&self.names),
+            varnames: map_names(&self.varnames),
+            cellvars: map_names(&self.cellvars),
+            freevars: map_names(&self.freevars),
+            source_path: bag.make_name(self.source_path.as_ref()),
+            obj_name: bag.make_name(self.obj_name.as_ref()),
+
+            instructions: self.instructions.clone(),
+            locations: self.locations.clone(),
+            flags: self.flags,
+            posonlyarg_count: self.posonlyarg_count,
+            arg_count: self.arg_count,
+            kwonlyarg_count: self.kwonlyarg_count,
+            first_line_number: self.first_line_number,
+            max_stackdepth: self.max_stackdepth,
+            cell2arg: self.cell2arg.clone(),
+        }
+    }
+}
+
+/// Error that occurs during code deserialization
+#[derive(Debug)]
+#[non_exhaustive]
+pub enum CodeDeserializeError {
+    /// Unexpected End Of File
+    Eof,
+    /// Invalid Bytecode
+    Other,
+}
+
+impl fmt::Display for CodeDeserializeError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Self::Eof => f.write_str("unexpected end of data"),
+            Self::Other => f.write_str("invalid bytecode"),
+        }
+    }
+}
+
+impl std::error::Error for CodeDeserializeError {}
+
+impl CodeObject<ConstantData> {
+    /// Load a code object from bytes
+    pub fn from_bytes(data: &[u8]) -> Result<Self, CodeDeserializeError> {
+        use lz4_flex::block::DecompressError;
+        let raw_bincode = lz4_flex::decompress_size_prepended(data).map_err(|e| match e {
+            DecompressError::OutputTooSmall { .. } | DecompressError::ExpectedAnotherByte => {
+                CodeDeserializeError::Eof
+            }
+            _ => CodeDeserializeError::Other,
+        })?;
+        let data = bincode::deserialize(&raw_bincode).map_err(|e| match *e {
+            bincode::ErrorKind::Io(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
+                CodeDeserializeError::Eof
+            }
+            _ => CodeDeserializeError::Other,
+        })?;
+        Ok(data)
+    }
+
+    /// Serialize this bytecode to bytes.
+    pub fn to_bytes(&self) -> Vec<u8> {
+        let data = bincode::serialize(&self).expect("CodeObject is not serializable");
+        lz4_flex::compress_prepend_size(&data)
+    }
+}
+
+impl<C: Constant> fmt::Display for CodeObject<C> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.display_inner(f, false, 1)?;
+        for constant in &*self.constants {
+            if let BorrowedConstant::Code { code } = constant.borrow_constant() {
+                writeln!(f, "\nDisassembly of {:?}", code)?;
+                code.fmt(f)?;
+            }
+        }
+        Ok(())
+    }
+}
+
+impl Instruction {
+    /// Gets the label stored inside this instruction, if it exists
+    #[inline]
+    pub fn label_arg(&self) -> Option<&Label> {
+        match self {
+            Jump { target: l }
+            | JumpIfTrue { target: l }
+            | JumpIfFalse { target: l }
+            | JumpIfTrueOrPop { target: l }
+            | JumpIfFalseOrPop { target: l }
+            | ForIter { target: l }
+            | SetupFinally { handler: l }
+            | SetupExcept { handler: l }
+            | SetupWith { end: l }
+            | SetupAsyncWith { end: l }
+            | SetupLoop { break_target: l }
+            | Continue { target: l } => Some(l),
+            _ => None,
+        }
+    }
+
+    /// Gets a mutable reference to the label stored inside this instruction, if it exists
+    #[inline]
+    pub fn label_arg_mut(&mut self) -> Option<&mut Label> {
+        match self {
+            Jump { target: l }
+            | JumpIfTrue { target: l }
+            | JumpIfFalse { target: l }
+            | JumpIfTrueOrPop { target: l }
+            | JumpIfFalseOrPop { target: l }
+            | ForIter { target: l }
+            | SetupFinally { handler: l }
+            | SetupExcept { handler: l }
+            | SetupWith { end: l }
+            | SetupAsyncWith { end: l }
+            | SetupLoop { break_target: l }
+            | Continue { target: l } => Some(l),
+            _ => None,
+        }
+    }
+
+    /// Whether this is an unconditional branching
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use rustpython_bytecode::{Instruction, Label};
+    /// let label = Label(0xF);
+    /// let jump_inst = Instruction::Jump {target: label};
+    /// assert!(jump_inst.unconditional_branch())
+    /// ```
+    pub fn unconditional_branch(&self) -> bool {
+        matches!(
+            self,
+            Jump { .. } | Continue { .. } | Break { .. } | ReturnValue | Raise { .. }
+        )
+    }
+
+    /// What effect this instruction has on the stack
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use rustpython_bytecode::{Instruction, Label, UnaryOperator};
+    /// let jump_instruction = Instruction::Jump {target: Label(0xF)};
+    /// let invert_instruction = Instruction::UnaryOperation {op: UnaryOperator::Invert};
+    /// assert_eq!(jump_instruction.stack_effect(true), 0);
+    /// assert_eq!(invert_instruction.stack_effect(false), 0);
+    /// ```
+    ///
+    pub fn stack_effect(&self, jump: bool) -> i32 {
+        match self {
+            ImportName { .. } | ImportNameless => -1,
+            ImportStar => -1,
+            ImportFrom { .. } => 1,
+            LoadFast(_) | LoadNameAny(_) | LoadGlobal(_) | LoadDeref(_) | LoadClassDeref(_) => 1,
+            StoreFast(_) | StoreLocal(_) | StoreGlobal(_) | StoreDeref(_) => -1,
+            DeleteFast(_) | DeleteLocal(_) | DeleteGlobal(_) | DeleteDeref(_) => 0,
+            LoadClosure(_) => 1,
+            Subscript => -1,
+            StoreSubscript => -3,
+            DeleteSubscript => -2,
+            LoadAttr { .. } => 0,
+            StoreAttr { .. } => -2,
+            DeleteAttr { .. } => -1,
+            LoadConst { .. } => 1,
+            UnaryOperation { .. } => 0,
+            BinaryOperation { .. }
+            | BinaryOperationInplace { .. }
+            | TestOperation { .. }
+            | CompareOperation { .. } => -1,
+            Pop => -1,
+            Rotate2 | Rotate3 => 0,
+            Duplicate => 1,
+            Duplicate2 => 2,
+            GetIter => 0,
+            Continue { .. } => 0,
+            Break { .. } => 0,
+            Jump { .. } => 0,
+            JumpIfTrue { .. } | JumpIfFalse { .. } => -1,
+            JumpIfTrueOrPop { .. } | JumpIfFalseOrPop { .. } => {
+                if jump {
+                    0
+                } else {
+                    -1
+                }
+            }
+            MakeFunction(flags) => {
+                -2 - flags.contains(MakeFunctionFlags::CLOSURE) as i32
+                    - flags.contains(MakeFunctionFlags::ANNOTATIONS) as i32
+                    - flags.contains(MakeFunctionFlags::KW_ONLY_DEFAULTS) as i32
+                    - flags.contains(MakeFunctionFlags::DEFAULTS) as i32
+                    + 1
+            }
+            CallFunctionPositional { nargs } => -(*nargs as i32) - 1 + 1,
+            CallMethodPositional { nargs } => -(*nargs as i32) - 3 + 1,
+            CallFunctionKeyword { nargs } => -1 - (*nargs as i32) - 1 + 1,
+            CallMethodKeyword { nargs } => -1 - (*nargs as i32) - 3 + 1,
+            CallFunctionEx { has_kwargs } => -1 - (*has_kwargs as i32) - 1 + 1,
+            CallMethodEx { has_kwargs } => -1 - (*has_kwargs as i32) - 3 + 1,
+            LoadMethod { .. } => -1 + 3,
+            ForIter { .. } => {
+                if jump {
+                    -1
+                } else {
+                    1
+                }
+            }
+            ReturnValue => -1,
+            YieldValue => 0,
+            YieldFrom => -1,
+            SetupAnnotation
+            | SetupLoop { .. }
+            | SetupFinally { .. }
+            | EnterFinally
+            | EndFinally => 0,
+            SetupExcept { .. } => {
+                if jump {
+                    1
+                } else {
+                    0
+                }
+            }
+            SetupWith { .. } => {
+                if jump {
+                    0
+                } else {
+                    1
+                }
+            }
+            WithCleanupStart => 0,
+            WithCleanupFinish => -1,
+            PopBlock => 0,
+            Raise { kind } => -(*kind as u8 as i32),
+            BuildString { size }
+            | BuildTuple { size, .. }
+            | BuildList { size, .. }
+            | BuildSet { size, .. } => -(*size as i32) + 1,
+            BuildMap { unpack, size, .. } => {
+                let nargs = if *unpack { *size } else { *size * 2 };
+                -(nargs as i32) + 1
+            }
+            DictUpdate => -1,
+            BuildSlice { step } => -2 - (*step as i32) + 1,
+            ListAppend { .. } | SetAdd { .. } => -1,
+            MapAdd { .. } => -2,
+            PrintExpr => -1,
+            LoadBuildClass => 1,
+            UnpackSequence { size } => -1 + *size as i32,
+            UnpackEx { before, after } => -1 + *before as i32 + 1 + *after as i32,
+            FormatValue { .. } => -1,
+            PopException => 0,
+            Reverse { .. } => 0,
+            GetAwaitable => 0,
+            BeforeAsyncWith => 1,
+            SetupAsyncWith { .. } => {
+                if jump {
+                    -1
+                } else {
+                    0
+                }
+            }
+            GetAIter => 0,
+            GetANext => 1,
+            EndAsyncFor => -2,
+        }
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    fn fmt_dis<C: Constant>(
+        &self,
+        f: &mut fmt::Formatter,
+        constants: &[C],
+        names: &[C::Name],
+        varnames: &[C::Name],
+        cellvars: &[C::Name],
+        freevars: &[C::Name],
+        expand_codeobjects: bool,
+        level: usize,
+    ) -> fmt::Result {
+        macro_rules! w {
+            ($variant:ident) => {
+                writeln!(f, stringify!($variant))
+            };
+            ($variant:ident, $var:expr) => {
+                writeln!(f, "{:20} ({})", stringify!($variant), $var)
+            };
+            ($variant:ident, $var1:expr, $var2:expr) => {
+                writeln!(f, "{:20} ({}, {})", stringify!($variant), $var1, $var2)
+            };
+            ($variant:ident, $var1:expr, $var2:expr, $var3:expr) => {
+                writeln!(
+                    f,
+                    "{:20} ({}, {}, {})",
+                    stringify!($variant),
+                    $var1,
+                    $var2,
+                    $var3
+                )
+            };
+        }
+
+        let varname = |i: u32| varnames[i as usize].as_ref();
+        let name = |i: u32| names[i as usize].as_ref();
+        let cellname = |i: u32| {
+            cellvars
+                .get(i as usize)
+                .unwrap_or_else(|| &freevars[i as usize - cellvars.len()])
+                .as_ref()
+        };
+
+        match self {
+            ImportName { idx } => w!(ImportName, name(*idx)),
+            ImportNameless => w!(ImportNameless),
+            ImportStar => w!(ImportStar),
+            ImportFrom { idx } => w!(ImportFrom, name(*idx)),
+            LoadFast(idx) => w!(LoadFast, *idx, varname(*idx)),
+            LoadNameAny(idx) => w!(LoadNameAny, *idx, name(*idx)),
+            LoadGlobal(idx) => w!(LoadGlobal, *idx, name(*idx)),
+            LoadDeref(idx) => w!(LoadDeref, *idx, cellname(*idx)),
+            LoadClassDeref(idx) => w!(LoadClassDeref, *idx, cellname(*idx)),
+            StoreFast(idx) => w!(StoreFast, *idx, varname(*idx)),
+            StoreLocal(idx) => w!(StoreLocal, *idx, name(*idx)),
+            StoreGlobal(idx) => w!(StoreGlobal, *idx, name(*idx)),
+            StoreDeref(idx) => w!(StoreDeref, *idx, cellname(*idx)),
+            DeleteFast(idx) => w!(DeleteFast, *idx, varname(*idx)),
+            DeleteLocal(idx) => w!(DeleteLocal, *idx, name(*idx)),
+            DeleteGlobal(idx) => w!(DeleteGlobal, *idx, name(*idx)),
+            DeleteDeref(idx) => w!(DeleteDeref, *idx, cellname(*idx)),
+            LoadClosure(i) => w!(LoadClosure, *i, cellname(*i)),
+            Subscript => w!(Subscript),
+            StoreSubscript => w!(StoreSubscript),
+            DeleteSubscript => w!(DeleteSubscript),
+            StoreAttr { idx } => w!(StoreAttr, name(*idx)),
+            DeleteAttr { idx } => w!(DeleteAttr, name(*idx)),
+            LoadConst { idx } => {
+                let value = &constants[*idx as usize];
+                match value.borrow_constant() {
+                    BorrowedConstant::Code { code } if expand_codeobjects => {
+                        writeln!(f, "{:20} ({:?}):", "LoadConst", code)?;
+                        code.display_inner(f, true, level + 1)?;
+                        Ok(())
+                    }
+                    c => {
+                        write!(f, "{:20} (", "LoadConst")?;
+                        c.fmt_display(f)?;
+                        writeln!(f, ")")
+                    }
+                }
+            }
+            UnaryOperation { op } => w!(UnaryOperation, format_args!("{:?}", op)),
+            BinaryOperation { op } => w!(BinaryOperation, format_args!("{:?}", op)),
+            BinaryOperationInplace { op } => {
+                w!(BinaryOperationInplace, format_args!("{:?}", op))
+            }
+            LoadAttr { idx } => w!(LoadAttr, name(*idx)),
+            TestOperation { op } => w!(TestOperation, format_args!("{:?}", op)),
+            CompareOperation { op } => w!(CompareOperation, format_args!("{:?}", op)),
+            Pop => w!(Pop),
+            Rotate2 => w!(Rotate2),
+            Rotate3 => w!(Rotate3),
+            Duplicate => w!(Duplicate),
+            Duplicate2 => w!(Duplicate2),
+            GetIter => w!(GetIter),
+            Continue { target } => w!(Continue, target),
+            Break { target } => w!(Break, target),
+            Jump { target } => w!(Jump, target),
+            JumpIfTrue { target } => w!(JumpIfTrue, target),
+            JumpIfFalse { target } => w!(JumpIfFalse, target),
+            JumpIfTrueOrPop { target } => w!(JumpIfTrueOrPop, target),
+            JumpIfFalseOrPop { target } => w!(JumpIfFalseOrPop, target),
+            MakeFunction(flags) => w!(MakeFunction, format_args!("{:?}", flags)),
+            CallFunctionPositional { nargs } => w!(CallFunctionPositional, nargs),
+            CallFunctionKeyword { nargs } => w!(CallFunctionKeyword, nargs),
+            CallFunctionEx { has_kwargs } => w!(CallFunctionEx, has_kwargs),
+            LoadMethod { idx } => w!(LoadMethod, name(*idx)),
+            CallMethodPositional { nargs } => w!(CallMethodPositional, nargs),
+            CallMethodKeyword { nargs } => w!(CallMethodKeyword, nargs),
+            CallMethodEx { has_kwargs } => w!(CallMethodEx, has_kwargs),
+            ForIter { target } => w!(ForIter, target),
+            ReturnValue => w!(ReturnValue),
+            YieldValue => w!(YieldValue),
+            YieldFrom => w!(YieldFrom),
+            SetupAnnotation => w!(SetupAnnotation),
+            SetupLoop { break_target } => w!(SetupLoop, break_target),
+            SetupExcept { handler } => w!(SetupExcept, handler),
+            SetupFinally { handler } => w!(SetupFinally, handler),
+            EnterFinally => w!(EnterFinally),
+            EndFinally => w!(EndFinally),
+            SetupWith { end } => w!(SetupWith, end),
+            WithCleanupStart => w!(WithCleanupStart),
+            WithCleanupFinish => w!(WithCleanupFinish),
+            BeforeAsyncWith => w!(BeforeAsyncWith),
+            SetupAsyncWith { end } => w!(SetupAsyncWith, end),
+            PopBlock => w!(PopBlock),
+            Raise { kind } => w!(Raise, format_args!("{:?}", kind)),
+            BuildString { size } => w!(BuildString, size),
+            BuildTuple { size, unpack } => w!(BuildTuple, size, unpack),
+            BuildList { size, unpack } => w!(BuildList, size, unpack),
+            BuildSet { size, unpack } => w!(BuildSet, size, unpack),
+            BuildMap {
+                size,
+                unpack,
+                for_call,
+            } => w!(BuildMap, size, unpack, for_call),
+            DictUpdate => w!(DictUpdate),
+            BuildSlice { step } => w!(BuildSlice, step),
+            ListAppend { i } => w!(ListAppend, i),
+            SetAdd { i } => w!(SetAdd, i),
+            MapAdd { i } => w!(MapAdd, i),
+            PrintExpr => w!(PrintExpr),
+            LoadBuildClass => w!(LoadBuildClass),
+            UnpackSequence { size } => w!(UnpackSequence, size),
+            UnpackEx { before, after } => w!(UnpackEx, before, after),
+            FormatValue { conversion } => w!(FormatValue, format_args!("{:?}", conversion)),
+            PopException => w!(PopException),
+            Reverse { amount } => w!(Reverse, amount),
+            GetAwaitable => w!(GetAwaitable),
+            GetAIter => w!(GetAIter),
+            GetANext => w!(GetANext),
+            EndAsyncFor => w!(EndAsyncFor),
+        }
+    }
+}
+
+impl fmt::Display for ConstantData {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.borrow_constant().fmt_display(f)
+    }
+}
+
+impl<C: Constant> fmt::Debug for CodeObject<C> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "<code object {} at ??? file {:?}, line {}>",
+            self.obj_name.as_ref(),
+            self.source_path.as_ref(),
+            self.first_line_number
+        )
+    }
+}
+
+/// A frozen module. Holds a code object and whether it is part of a package
+#[derive(Serialize, Deserialize, Debug)]
+pub struct FrozenModule {
+    pub code: CodeObject<ConstantData>,
+    pub package: bool,
+}
+
+pub mod frozen_lib {
+    use super::*;
+    use bincode::{options, Options};
+    use std::io;
+
+    /// Decode a library to a iterable of frozen modules
+    pub fn decode_lib(bytes: &[u8]) -> FrozenModulesIter {
+        let data = lz4_flex::decompress_size_prepended(bytes).unwrap();
+        let r = VecReader { data, pos: 0 };
+        let mut de = bincode::Deserializer::with_bincode_read(r, options());
+        let len = u64::deserialize(&mut de).unwrap().try_into().unwrap();
+        FrozenModulesIter { len, de }
+    }
+
+    pub struct FrozenModulesIter {
+        len: usize,
+        // ideally this could be a SeqAccess, but I think that would require existential types
+        de: bincode::Deserializer<VecReader, bincode::DefaultOptions>,
+    }
+
+    impl Iterator for FrozenModulesIter {
+        type Item = (String, FrozenModule);
+
+        fn next(&mut self) -> Option<Self::Item> {
+            // manually mimic bincode's seq encoding, which is <len:u64> <element*len>
+            // This probably won't change (bincode doesn't require padding or anything), but
+            // it's not guaranteed by semver as far as I can tell
+            if self.len > 0 {
+                let entry = Deserialize::deserialize(&mut self.de).unwrap();
+                self.len -= 1;
+                Some(entry)
+            } else {
+                None
+            }
+        }
+
+        fn size_hint(&self) -> (usize, Option<usize>) {
+            (self.len, Some(self.len))
+        }
+    }
+
+    impl ExactSizeIterator for FrozenModulesIter {}
+
+    /// Encode the given iterator of frozen modules into a compressed vector of bytes
+    pub fn encode_lib<'a, I>(lib: I) -> Vec<u8>
+    where
+        I: IntoIterator<Item = (&'a str, &'a FrozenModule)>,
+        I::IntoIter: ExactSizeIterator + Clone,
+    {
+        let iter = lib.into_iter();
+        let data = options().serialize(&SerializeLib { iter }).unwrap();
+        lz4_flex::compress_prepend_size(&data)
+    }
+
+    struct SerializeLib<I> {
+        iter: I,
+    }
+
+    impl<'a, I> Serialize for SerializeLib<I>
+    where
+        I: ExactSizeIterator<Item = (&'a str, &'a FrozenModule)> + Clone,
+    {
+        fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+        where
+            S: serde::Serializer,
+        {
+            serializer.collect_seq(self.iter.clone())
+        }
+    }
+
+    /// Owned version of bincode::de::read::SliceReader<'a>
+    struct VecReader {
+        data: Vec<u8>,
+        pos: usize,
+    }
+
+    impl io::Read for VecReader {
+        fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+            let mut subslice = &self.data[self.pos..];
+            let n = io::Read::read(&mut subslice, buf)?;
+            self.pos += n;
+            Ok(n)
+        }
+        fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
+            self.get_byte_slice(buf.len())
+                .map(|data| buf.copy_from_slice(data))
+        }
+    }
+
+    impl VecReader {
+        #[inline(always)]
+        fn get_byte_slice(&mut self, length: usize) -> io::Result<&[u8]> {
+            let subslice = &self.data[self.pos..];
+            match subslice.get(..length) {
+                Some(ret) => {
+                    self.pos += length;
+                    Ok(ret)
+                }
+                None => Err(io::ErrorKind::UnexpectedEof.into()),
+            }
+        }
+    }
+
+    impl<'storage> bincode::BincodeRead<'storage> for VecReader {
+        fn forward_read_str<V>(&mut self, length: usize, visitor: V) -> bincode::Result<V::Value>
+        where
+            V: serde::de::Visitor<'storage>,
+        {
+            let bytes = self.get_byte_slice(length)?;
+            match ::std::str::from_utf8(bytes) {
+                Ok(s) => visitor.visit_str(s),
+                Err(e) => Err(bincode::ErrorKind::InvalidUtf8Encoding(e).into()),
+            }
+        }
+
+        fn get_byte_buffer(&mut self, length: usize) -> bincode::Result<Vec<u8>> {
+            self.get_byte_slice(length)
+                .map(|x| x.to_vec())
+                .map_err(Into::into)
+        }
+
+        fn forward_read_bytes<V>(&mut self, length: usize, visitor: V) -> bincode::Result<V::Value>
+        where
+            V: serde::de::Visitor<'storage>,
+        {
+            visitor.visit_bytes(self.get_byte_slice(length)?)
+        }
+    }
+}
diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml
new file mode 100644
index 00000000..3a770df3
--- /dev/null
+++ b/codegen/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+name = "rustpython-codegen"
+version = "0.1.2"
+description = "Compiler for python code into bytecode for the rustpython VM."
+authors = ["RustPython Team"]
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
+edition = "2021"
+
+[dependencies]
+indexmap = "1.8.1"
+itertools = "0.10.3"
+rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
+rustpython-ast = { path = "../ast", features = ["unparse"] }
+num-complex = { version = "0.4.0", features = ["serde"] }
+num-traits = "0.2.14"
+log = "0.4.16"
+ahash = "0.7.6"
+
+[dev-dependencies]
+rustpython-parser = { path = "../parser" }
+insta = "1.14.0"
diff --git a/src/compile.rs b/codegen/src/compile.rs
similarity index 100%
rename from src/compile.rs
rename to codegen/src/compile.rs
diff --git a/src/error.rs b/codegen/src/error.rs
similarity index 100%
rename from src/error.rs
rename to codegen/src/error.rs
diff --git a/src/ir.rs b/codegen/src/ir.rs
similarity index 100%
rename from src/ir.rs
rename to codegen/src/ir.rs
diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs
new file mode 100644
index 00000000..3e3aca8f
--- /dev/null
+++ b/codegen/src/lib.rs
@@ -0,0 +1,15 @@
+//! Compile a Python AST or source code into bytecode consumable by RustPython.
+#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
+#![doc(html_root_url = "https://docs.rs/rustpython-compiler/")]
+
+#[macro_use]
+extern crate log;
+
+type IndexMap<K, V> = indexmap::IndexMap<K, V, ahash::RandomState>;
+type IndexSet<T> = indexmap::IndexSet<T, ahash::RandomState>;
+
+pub mod compile;
+pub mod error;
+pub mod ir;
+pub mod mode;
+pub mod symboltable;
diff --git a/src/mode.rs b/codegen/src/mode.rs
similarity index 100%
rename from src/mode.rs
rename to codegen/src/mode.rs
diff --git a/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap b/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap
new file mode 100644
index 00000000..93eb0ea6
--- /dev/null
+++ b/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap
@@ -0,0 +1,14 @@
+---
+source: compiler/codegen/src/compile.rs
+expression: "compile_exec(\"\\\nif True and False and False:\n    pass\n\")"
+---
+  1           0 LoadConst            (True)
+              1 JumpIfFalse          (6)
+              2 LoadConst            (False)
+              3 JumpIfFalse          (6)
+              4 LoadConst            (False)
+              5 JumpIfFalse          (6)
+
+  2     >>    6 LoadConst            (None)
+              7 ReturnValue
+
diff --git a/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap b/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap
new file mode 100644
index 00000000..9594af69
--- /dev/null
+++ b/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap
@@ -0,0 +1,16 @@
+---
+source: compiler/codegen/src/compile.rs
+expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n    pass\n\")"
+---
+  1           0 LoadConst            (True)
+              1 JumpIfFalse          (4)
+              2 LoadConst            (False)
+              3 JumpIfTrue           (8)
+        >>    4 LoadConst            (False)
+              5 JumpIfFalse          (8)
+              6 LoadConst            (True)
+              7 JumpIfFalse          (8)
+
+  2     >>    8 LoadConst            (None)
+              9 ReturnValue
+
diff --git a/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap b/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap
new file mode 100644
index 00000000..dd582b7d
--- /dev/null
+++ b/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap
@@ -0,0 +1,14 @@
+---
+source: compiler/codegen/src/compile.rs
+expression: "compile_exec(\"\\\nif True or False or False:\n    pass\n\")"
+---
+  1           0 LoadConst            (True)
+              1 JumpIfTrue           (6)
+              2 LoadConst            (False)
+              3 JumpIfTrue           (6)
+              4 LoadConst            (False)
+              5 JumpIfFalse          (6)
+
+  2     >>    6 LoadConst            (None)
+              7 ReturnValue
+
diff --git a/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap b/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap
new file mode 100644
index 00000000..33180c7c
--- /dev/null
+++ b/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap
@@ -0,0 +1,87 @@
+---
+source: compiler/codegen/src/compile.rs
+expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n    with self.subTest(type=type(stop_exc)):\n        try:\n            async with woohoo():\n                raise stop_exc\n        except Exception as ex:\n            self.assertIs(ex, stop_exc)\n        else:\n            self.fail(f'{stop_exc} was suppressed')\n\")"
+---
+  1           0 SetupLoop            (69)
+              1 LoadNameAny          (0, StopIteration)
+              2 LoadConst            ("spam")
+              3 CallFunctionPositional (1)
+              4 LoadNameAny          (1, StopAsyncIteration)
+              5 LoadConst            ("ham")
+              6 CallFunctionPositional (1)
+              7 BuildTuple           (2, false)
+              8 GetIter
+        >>    9 ForIter              (68)
+             10 StoreLocal           (2, stop_exc)
+
+  2          11 LoadNameAny          (3, self)
+             12 LoadMethod           (subTest)
+             13 LoadNameAny          (5, type)
+             14 LoadNameAny          (2, stop_exc)
+             15 CallFunctionPositional (1)
+             16 LoadConst            (("type"))
+             17 CallMethodKeyword    (1)
+             18 SetupWith            (65)
+             19 Pop
+
+  3          20 SetupExcept          (40)
+
+  4          21 LoadNameAny          (6, woohoo)
+             22 CallFunctionPositional (0)
+             23 BeforeAsyncWith
+             24 GetAwaitable
+             25 LoadConst            (None)
+             26 YieldFrom
+             27 SetupAsyncWith       (33)
+             28 Pop
+
+  5          29 LoadNameAny          (2, stop_exc)
+             30 Raise                (Raise)
+
+  4          31 PopBlock
+             32 EnterFinally
+        >>   33 WithCleanupStart
+             34 GetAwaitable
+             35 LoadConst            (None)
+             36 YieldFrom
+             37 WithCleanupFinish
+             38 PopBlock
+             39 Jump                 (54)
+        >>   40 Duplicate
+
+  6          41 LoadNameAny          (7, Exception)
+             42 TestOperation        (ExceptionMatch)
+             43 JumpIfFalse          (53)
+             44 StoreLocal           (8, ex)
+
+  7          45 LoadNameAny          (3, self)
+             46 LoadMethod           (assertIs)
+             47 LoadNameAny          (8, ex)
+             48 LoadNameAny          (2, stop_exc)
+             49 CallMethodPositional (2)
+             50 Pop
+             51 PopException
+             52 Jump                 (63)
+        >>   53 Raise                (Reraise)
+
+  9     >>   54 LoadNameAny          (3, self)
+             55 LoadMethod           (fail)
+             56 LoadConst            ("")
+
+  1          57 LoadNameAny          (2, stop_exc)
+             58 FormatValue          (None)
+
+  9          59 LoadConst            (" was suppressed")
+             60 BuildString          (2)
+             61 CallMethodPositional (1)
+             62 Pop
+
+  2     >>   63 PopBlock
+             64 EnterFinally
+        >>   65 WithCleanupStart
+             66 WithCleanupFinish
+             67 Jump                 (9)
+        >>   68 PopBlock
+        >>   69 LoadConst            (None)
+             70 ReturnValue
+
diff --git a/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap b/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap
similarity index 100%
rename from src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap
rename to codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap
diff --git a/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap b/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap
similarity index 100%
rename from src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap
rename to codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap
diff --git a/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap b/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap
similarity index 100%
rename from src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap
rename to codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap
diff --git a/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap b/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
similarity index 100%
rename from src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
rename to codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap
diff --git a/src/symboltable.rs b/codegen/src/symboltable.rs
similarity index 100%
rename from src/symboltable.rs
rename to codegen/src/symboltable.rs
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
new file mode 100644
index 00000000..60dabdcc
--- /dev/null
+++ b/parser/Cargo.toml
@@ -0,0 +1,31 @@
+[package]
+name = "rustpython-parser"
+version = "0.1.2"
+description = "Parser for python code."
+authors = [ "RustPython Team" ]
+build = "build.rs"
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
+edition = "2021"
+
+[build-dependencies]
+anyhow = "1.0.45"
+lalrpop = { version = "0.19.8", optional = true }
+phf_codegen = "0.10"
+tiny-keccak = { version = "2", features = ["sha3"] }
+
+[dependencies]
+ahash = "0.7.6"
+itertools = "0.10.3"
+lalrpop-util = "0.19.8"
+log = "0.4.16"
+num-bigint = "0.4.3"
+num-traits = "0.2.14"
+phf = "0.10.1"
+rustpython-ast = { path = "../ast" }
+unic-emoji-char = "0.9.0"
+unic-ucd-ident  = "0.9.0"
+unicode_names2 = "0.5.0"
+
+[dev-dependencies]
+insta = "1.14.0"
diff --git a/parser/README.md b/parser/README.md
new file mode 100644
index 00000000..ebe9eb68
--- /dev/null
+++ b/parser/README.md
@@ -0,0 +1,66 @@
+# RustPython/parser
+
+This directory has the code for python lexing, parsing and generating Abstract Syntax Trees (AST).
+
+The steps are:
+- Lexical analysis: splits the source code into tokens.
+- Parsing and generating the AST: transforms those tokens into an AST. Uses `LALRPOP`, a Rust parser generator framework.
+
+This crate is published on [https://docs.rs/rustpython-parser](https://docs.rs/rustpython-parser).
+
+We wrote [a blog post](https://rustpython.github.io/2020/04/02/thing-explainer-parser.html) with screenshots and an explanation to help you understand the steps by seeing them in action.
+
+For more information on LALRPOP, here is a link to the [LALRPOP book](https://github.com/lalrpop/lalrpop).
+
+There is a readme in the `src` folder with the details of each file.
+
+
+## Directory content
+
+`build.rs`: The build script.
+`Cargo.toml`: The config file.
+
+The `src` directory has:
+
+**lib.rs**   
+This is the crate's root.
+
+**lexer.rs**   
+This module takes care of lexing python source text. This means source code is translated into separate tokens.
+
+**parser.rs**   
+A python parsing module. Use this module to parse python code into an AST. There are three ways to parse python code. You could parse a whole program, a single statement, or a single expression.
+
+**ast.rs**   
+ Implements abstract syntax tree (AST) nodes for the python language. Roughly equivalent to [the python AST](https://docs.python.org/3/library/ast.html).
+
+**python.lalrpop**   
+Python grammar.
+
+**token.rs**   
+Different token definitions. Loosely based on token.h from CPython source.
+
+**errors.rs**   
+Define internal parse error types. The goal is to provide a matching and a safe error API, masking errors from LALR.
+
+**fstring.rs**   
+Format strings.
+
+**function.rs**   
+Collection of functions for parsing parameters, arguments.
+
+**location.rs**   
+Datatypes to support source location information.
+
+**mode.rs**   
+Execution mode check. Allowed modes are `exec`, `eval` or `single`.
+
+
+## How to use
+
+For example, one could do this:
+```
+  use rustpython_parser::{parser, ast};
+  let python_source = "print('Hello world')";
+  let python_ast = parser::parse_expression(python_source).unwrap();
+```
diff --git a/parser/build.rs b/parser/build.rs
new file mode 100644
index 00000000..874b0166
--- /dev/null
+++ b/parser/build.rs
@@ -0,0 +1,143 @@
+use std::fmt::Write as _;
+use std::fs::File;
+use std::io::{BufRead, BufReader, BufWriter, Write};
+use std::path::PathBuf;
+use tiny_keccak::{Hasher, Sha3};
+
+fn main() -> anyhow::Result<()> {
+    const SOURCE: &str = "python.lalrpop";
+    const TARGET: &str = "python.rs";
+
+    println!("cargo:rerun-if-changed={SOURCE}");
+
+    try_lalrpop(SOURCE, TARGET)?;
+    gen_phf();
+
+    Ok(())
+}
+
+fn requires_lalrpop(source: &str, target: &str) -> bool {
+    let target = if let Ok(target) = File::open(target) {
+        target
+    } else {
+        println!("cargo:warning=python.rs doesn't exist. regenerate.");
+        return true;
+    };
+
+    let sha_prefix = "// sha3: ";
+    let sha3_line = BufReader::with_capacity(128, target)
+        .lines()
+        .find_map(|line| {
+            let line = line.unwrap();
+            line.starts_with(sha_prefix).then_some(line)
+        })
+        .expect("no sha3 line?");
+    let expected_sha3_str = sha3_line.strip_prefix(sha_prefix).unwrap();
+
+    let actual_sha3 = {
+        let mut hasher = Sha3::v256();
+        let mut f = BufReader::new(File::open(source).unwrap());
+        let mut line = String::new();
+        while f.read_line(&mut line).unwrap() != 0 {
+            if line.ends_with('\n') {
+                line.pop();
+                if line.ends_with('\r') {
+                    line.pop();
+                }
+            }
+            hasher.update(line.as_bytes());
+            hasher.update(b"\n");
+            line.clear();
+        }
+        let mut hash = [0u8; 32];
+        hasher.finalize(&mut hash);
+        hash
+    };
+    let eq = sha_equal(expected_sha3_str, &actual_sha3);
+    if !eq {
+        println!("cargo:warning=python.rs hash expected: {expected_sha3_str}");
+        let mut actual_sha3_str = String::new();
+        for byte in actual_sha3 {
+            write!(actual_sha3_str, "{byte:02x}").unwrap();
+        }
+        println!("cargo:warning=python.rs hash   actual: {actual_sha3_str}");
+    }
+    !eq
+}
+
+fn try_lalrpop(source: &str, target: &str) -> anyhow::Result<()> {
+    if !requires_lalrpop(source, target) {
+        return Ok(());
+    }
+
+    #[cfg(feature = "lalrpop")]
+    {
+        lalrpop::process_root().expect("running lalrpop failed");
+        Ok(())
+    }
+
+    #[cfg(not(feature = "lalrpop"))]
+    panic!("try: cargo build --manifest-path=compiler/parser/Cargo.toml --features=lalrpop");
+}
+
+fn sha_equal(expected_sha3_str: &str, actual_sha3: &[u8; 32]) -> bool {
+    if expected_sha3_str.len() != 64 {
+        panic!("lalrpop version? hash bug is fixed in 0.19.8");
+    }
+
+    let mut expected_sha3 = [0u8; 32];
+    for (i, b) in expected_sha3.iter_mut().enumerate() {
+        *b = u8::from_str_radix(&expected_sha3_str[i * 2..][..2], 16).unwrap();
+    }
+    *actual_sha3 == expected_sha3
+}
+
+fn gen_phf() {
+    let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
+    let mut kwds = phf_codegen::Map::new();
+    let kwds = kwds
+        // Alphabetical keywords:
+        .entry("...", "Tok::Ellipsis")
+        .entry("False", "Tok::False")
+        .entry("None", "Tok::None")
+        .entry("True", "Tok::True")
+        // moreso "standard" keywords
+        .entry("and", "Tok::And")
+        .entry("as", "Tok::As")
+        .entry("assert", "Tok::Assert")
+        .entry("async", "Tok::Async")
+        .entry("await", "Tok::Await")
+        .entry("break", "Tok::Break")
+        .entry("class", "Tok::Class")
+        .entry("continue", "Tok::Continue")
+        .entry("def", "Tok::Def")
+        .entry("del", "Tok::Del")
+        .entry("elif", "Tok::Elif")
+        .entry("else", "Tok::Else")
+        .entry("except", "Tok::Except")
+        .entry("finally", "Tok::Finally")
+        .entry("for", "Tok::For")
+        .entry("from", "Tok::From")
+        .entry("global", "Tok::Global")
+        .entry("if", "Tok::If")
+        .entry("import", "Tok::Import")
+        .entry("in", "Tok::In")
+        .entry("is", "Tok::Is")
+        .entry("lambda", "Tok::Lambda")
+        .entry("nonlocal", "Tok::Nonlocal")
+        .entry("not", "Tok::Not")
+        .entry("or", "Tok::Or")
+        .entry("pass", "Tok::Pass")
+        .entry("raise", "Tok::Raise")
+        .entry("return", "Tok::Return")
+        .entry("try", "Tok::Try")
+        .entry("while", "Tok::While")
+        .entry("with", "Tok::With")
+        .entry("yield", "Tok::Yield")
+        .build();
+    writeln!(
+        BufWriter::new(File::create(out_dir.join("keywords.rs")).unwrap()),
+        "{kwds}",
+    )
+    .unwrap();
+}
diff --git a/parser/python.lalrpop b/parser/python.lalrpop
new file mode 100644
index 00000000..d72dcc9f
--- /dev/null
+++ b/parser/python.lalrpop
@@ -0,0 +1,1352 @@
+// See also: file:///usr/share/doc/python/html/reference/grammar.html?highlight=grammar
+// See also: https://github.com/antlr/grammars-v4/blob/master/python3/Python3.g4
+// See also: file:///usr/share/doc/python/html/reference/compound_stmts.html#function-definitions
+// See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword
+
+use crate::{
+    ast,
+    error::{LexicalError, LexicalErrorType},
+    function::{ArgumentList, parse_args, parse_params},
+    lexer,
+    string::parse_strings,
+    token::StringKind
+};
+use num_bigint::BigInt;
+
+grammar;
+
+// This is a hack to reduce the amount of lalrpop tables generated:
+// For each public entry point, a full parse table is generated.
+// By having only a single pub function, we reduce this to one.
+pub Top: ast::Mod = {
+    StartModule <body:Program> => ast::Mod::Module { body, type_ignores: vec![] },
+    StartInteractive <body:Program> => ast::Mod::Interactive { body },
+    StartExpression <body:TestList> ("\n")* => ast::Mod::Expression { body: Box::new(body) },
+};
+
+Program: ast::Suite = {
+    <lines:FileLine*> => {
+        lines.into_iter().flatten().collect()
+    },
+};
+
+// A file line either has a declaration, or an empty newline:
+FileLine: ast::Suite = {
+    Statement,
+    "\n" => vec![],
+};
+
+Suite: ast::Suite = {
+    SimpleStatement,
+    "\n" Indent <s:Statement+> Dedent => s.into_iter().flatten().collect(),
+};
+
+Statement: ast::Suite = {
+    SimpleStatement,
+    <s:CompoundStatement> => vec![s],
+};
+
+SimpleStatement: ast::Suite = {
+    <s1:SmallStatement> <s2:(";" SmallStatement)*> ";"? "\n" => {
+        let mut statements = vec![s1];
+        statements.extend(s2.into_iter().map(|e| e.1));
+        statements
+    }
+};
+
+SmallStatement: ast::Stmt = {
+    ExpressionStatement,
+    PassStatement,
+    DelStatement,
+    FlowStatement,
+    ImportStatement,
+    GlobalStatement,
+    NonlocalStatement,
+    AssertStatement,
+};
+
+PassStatement: ast::Stmt = {
+    <location:@L> "pass" => {
+        ast::Stmt {
+            location,
+            custom: (),
+            node: ast::StmtKind::Pass,
+        }
+    },
+};
+
+DelStatement: ast::Stmt = {
+    <location:@L> "del" <targets:ExpressionList2> => {
+        ast::Stmt {
+            location,
+            custom: (),
+            node: ast::StmtKind::Delete { targets },
+        }
+    },
+};
+
+ExpressionStatement: ast::Stmt = {
+    <location:@L> <expression:TestOrStarExprList> <suffix:AssignSuffix*> => {
+        // Just an expression, no assignment:
+        if suffix.is_empty() {
+            ast::Stmt {
+                custom: (),
+                location,
+                node: ast::StmtKind::Expr { value: Box::new(expression) }
+            }
+        } else {
+            let mut targets = vec![expression];
+            let mut values = suffix;
+
+            while values.len() > 1 {
+                targets.push(values.remove(0));
+            }
+
+            let value = Box::new(values.into_iter().next().unwrap());
+
+            ast::Stmt {
+                custom: (),
+                location,
+                node: ast::StmtKind::Assign { targets, value, type_comment: None },
+            }
+        }
+    },
+    <location:@L> <target:TestOrStarExprList> <op:AugAssign> <rhs:TestListOrYieldExpr> => {
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::AugAssign {
+                target: Box::new(target),
+                op,
+                value: Box::new(rhs)
+            },
+        }
+    },
+    <location:@L> <target:Test> ":" <annotation:Test> <rhs:AssignSuffix?> => {
+        let simple = matches!(target.node, ast::ExprKind::Name { .. });
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::AnnAssign {
+                target: Box::new(target),
+                annotation: Box::new(annotation),
+                value: rhs.map(Box::new),
+                simple: if simple { 1 } else { 0 },
+            },
+        }
+    },
+};
+
+AssignSuffix: ast::Expr = {
+    "=" <e:TestListOrYieldExpr> => e
+};
+
+TestListOrYieldExpr: ast::Expr = {
+    TestList,
+    YieldExpr
+}
+
+#[inline]
+TestOrStarExprList: ast::Expr = {
+    // as far as I can tell, these were the same
+    TestList
+};
+
+TestOrStarNamedExprList: ast::Expr = {
+    GenericList<TestOrStarNamedExpr>
+};
+
+TestOrStarExpr: ast::Expr = {
+    Test,
+    StarExpr,
+};
+
+TestOrStarNamedExpr: ast::Expr = {
+    NamedExpressionTest,
+    StarExpr,
+};
+
+AugAssign: ast::Operator = {
+    "+=" => ast::Operator::Add,
+    "-=" => ast::Operator::Sub,
+    "*=" => ast::Operator::Mult,
+    "@=" => ast::Operator::MatMult,
+    "/=" => ast::Operator::Div,
+    "%=" => ast::Operator::Mod,
+    "&=" => ast::Operator::BitAnd,
+    "|=" => ast::Operator::BitOr,
+    "^=" => ast::Operator::BitXor,
+    "<<=" => ast::Operator::LShift,
+    ">>=" => ast::Operator::RShift,
+    "**=" => ast::Operator::Pow,
+    "//=" => ast::Operator::FloorDiv,
+};
+
+FlowStatement: ast::Stmt = {
+    <location:@L> "break" => {
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::Break,
+        }
+    },
+    <location:@L> "continue" => {
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::Continue,
+        }
+    },
+    <location:@L> "return" <value:TestList?> => {
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::Return { value: value.map(Box::new) },
+        }
+    },
+    <location:@L> <expression:YieldExpr> => {
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::Expr { value: Box::new(expression) },
+        }
+    },
+    RaiseStatement,
+};
+
+RaiseStatement: ast::Stmt = {
+    <location:@L> "raise" => {
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::Raise { exc: None, cause: None },
+        }
+    },
+    <location:@L> "raise" <t:Test> <c:("from" Test)?> => {
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::Raise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)) },
+        }
+    },
+};
+
+ImportStatement: ast::Stmt = {
+    <location:@L> "import" <names: OneOrMore<ImportAsAlias<DottedName>>> => {
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::Import { names },
+        }
+    },
+    <location:@L> "from" <source:ImportFromLocation> "import" <names: ImportAsNames> => {
+        let (level, module) = source;
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::ImportFrom {
+                level,
+                module,
+                names
+            },
+        }
+    },
+};
+
+ImportFromLocation: (Option<usize>, Option<String>) = {
+    <dots: ImportDots*> <name:DottedName> => {
+        (Some(dots.iter().sum()), Some(name))
+    },
+    <dots: ImportDots+> => {
+        (Some(dots.iter().sum()), None)
+    },
+};
+
+ImportDots: usize = {
+    "..." => 3,
+    "." => 1,
+};
+
+ImportAsNames: Vec<ast::Alias> = {
+    <location:@L> <i:OneOrMore<ImportAsAlias<Identifier>>> => i,
+    <location:@L> "(" <i:OneOrMore<ImportAsAlias<Identifier>>> ","? ")" => i,
+    <location:@L> "*" => {
+        // Star import all
+        vec![ast::Alias::new(location, ast::AliasData { name: "*".to_string(), asname: None })]
+    },
+};
+
+
+#[inline]
+ImportAsAlias<I>: ast::Alias = {
+    <location:@L> <name:I> <a: ("as" Identifier)?> => ast::Alias::new(location, ast::AliasData { name, asname: a.map(|a| a.1) }),
+}
+
+// A name like abc or abc.def.ghi
+DottedName: String = {
+    <n:name> => n,
+    <n:name> <n2: ("." Identifier)+>  => {
+        let mut r = n.to_string();
+        for x in n2 {
+            r.push_str(".");
+            r.push_str(&x.1);
+        }
+        r
+    },
+};
+
+GlobalStatement: ast::Stmt = {
+    <location:@L> "global" <names:OneOrMore<Identifier>> => {
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::Global { names }
+        }
+    },
+};
+
+NonlocalStatement: ast::Stmt = {
+    <location:@L> "nonlocal" <names:OneOrMore<Identifier>> => {
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::Nonlocal { names }
+        }
+    },
+};
+
+AssertStatement: ast::Stmt = {
+    <location:@L> "assert" <test:Test> <msg: ("," Test)?> => {
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::Assert {
+                test: Box::new(test),
+                msg: msg.map(|e| Box::new(e.1))
+            }
+        }
+    },
+};
+
+CompoundStatement: ast::Stmt = {
+    IfStatement,
+    WhileStatement,
+    ForStatement,
+    TryStatement,
+    WithStatement,
+    FuncDef,
+    ClassDef,
+};
+
+IfStatement: ast::Stmt = {
+    <location:@L> "if" <test:NamedExpressionTest> ":" <body:Suite> <s2:(@L "elif" NamedExpressionTest ":" Suite)*> <s3:("else" ":" Suite)?> => {
+        // Determine last else:
+        let mut last = s3.map(|s| s.2).unwrap_or_default();
+
+        // handle elif:
+        for i in s2.into_iter().rev() {
+            let x = ast::Stmt {
+                custom: (),
+                location: i.0,
+                node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last },
+            };
+            last = vec![x];
+        }
+
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::If { test: Box::new(test), body, orelse: last }
+        }
+    },
+};
+
+WhileStatement: ast::Stmt = {
+    <location:@L> "while" <test:NamedExpressionTest> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
+        let orelse = s2.map(|s| s.2).unwrap_or_default();
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::While {
+                test: Box::new(test),
+                body,
+                orelse
+            },
+        }
+    },
+};
+
+ForStatement: ast::Stmt = {
+    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
+        let orelse = s2.map(|s| s.2).unwrap_or_default();
+        let target = Box::new(target);
+        let iter = Box::new(iter);
+        let type_comment = None;
+        let node = if is_async.is_some() {
+            ast::StmtKind::AsyncFor { target, iter, body, orelse, type_comment }
+        } else {
+            ast::StmtKind::For { target, iter, body, orelse, type_comment }
+        };
+        ast::Stmt::new(location, node)
+    },
+};
+
+TryStatement: ast::Stmt = {
+    <location:@L> "try" ":" <body:Suite> <handlers:ExceptClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> => {
+        let orelse = else_suite.map(|s| s.2).unwrap_or_default();
+        let finalbody = finally.map(|s| s.2).unwrap_or_default();
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::Try {
+                body,
+                handlers,
+                orelse,
+                finalbody,
+            },
+        }
+    },
+    <location:@L> "try" ":" <body:Suite> <finally:("finally" ":" Suite)> => {
+        let handlers = vec![];
+        let orelse = vec![];
+        let finalbody = finally.2;
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::Try {
+                body,
+                handlers,
+                orelse,
+                finalbody,
+            },
+        }
+    },
+};
+
+ExceptClause: ast::Excepthandler = {
+    <location:@L> "except" <typ:Test?> ":" <body:Suite> => {
+        ast::Excepthandler::new(
+            location,
+            ast::ExcepthandlerKind::ExceptHandler {
+                type_: typ.map(Box::new),
+                name: None,
+                body,
+            },
+        )
+    },
+    <location:@L> "except" <x:(Test "as" Identifier)> ":" <body:Suite> => {
+        ast::Excepthandler::new(
+            location,
+            ast::ExcepthandlerKind::ExceptHandler {
+                type_: Some(Box::new(x.0)),
+                name: Some(x.2),
+                body,
+            },
+        )
+    },
+};
+
+WithStatement: ast::Stmt = {
+    <location:@L> <is_async:"async"?> "with" <items:OneOrMore<WithItem>> ":" <body:Suite> => {
+        let type_comment = None;
+        let node = if is_async.is_some() {
+            ast::StmtKind::AsyncWith { items, body, type_comment }
+        } else {
+            ast::StmtKind::With { items, body, type_comment }
+        };
+        ast::Stmt::new(location, node)
+    },
+};
+
+WithItem: ast::Withitem = {
+    <context_expr:Test> <n:("as" Expression)?> => {
+        let optional_vars = n.map(|val| Box::new(val.1));
+        let context_expr = Box::new(context_expr);
+        ast::Withitem { context_expr, optional_vars }
+    },
+};
+
+FuncDef: ast::Stmt = {
+    <decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test)?> ":" <body:Suite> => {
+        let args = Box::new(args);
+        let returns = r.map(|x| Box::new(x.1));
+        let type_comment = None;
+        let node = if is_async.is_some() {
+            ast::StmtKind::AsyncFunctionDef { name, args, body, decorator_list, returns, type_comment }
+        } else {
+            ast::StmtKind::FunctionDef { name, args, body, decorator_list, returns, type_comment }
+        };
+        ast::Stmt::new(location, node)
+    },
+};
+
+Parameters: ast::Arguments = {
+    "(" <a: (ParameterList<TypedParameter>)?> ")" => {
+        a.unwrap_or_else(|| ast::Arguments {
+            posonlyargs: vec![],
+            args: vec![],
+            vararg: None,
+            kwonlyargs: vec![],
+            kw_defaults: vec![],
+            kwarg: None,
+            defaults: vec![]
+        })
+    }
+};
+
+// Note that this is a macro which is used once for function defs, and
+// once for lambda defs.
+ParameterList<ArgType>: ast::Arguments = {
+    <param1:ParameterDefs<ArgType>> <args2:("," ParameterListStarArgs<ArgType>)?> ","? =>? {
+        let (posonlyargs, args, defaults) = parse_params(param1)?;
+
+        // Now gather rest of parameters:
+        let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1);
+
+        Ok(ast::Arguments {
+            posonlyargs,
+            args,
+            kwonlyargs,
+            vararg,
+            kwarg,
+            defaults,
+            kw_defaults,
+        })
+    },
+    <param1:ParameterDefs<ArgType>> <kw:("," KwargParameter<ArgType>)> ","? =>? {
+        let (posonlyargs, args, defaults) = parse_params(param1)?;
+
+        // Now gather rest of parameters:
+        let vararg = None;
+        let kwonlyargs = vec![];
+        let kw_defaults = vec![];
+        let kwarg = kw.1;
+
+        Ok(ast::Arguments {
+            posonlyargs,
+            args,
+            kwonlyargs,
+            vararg,
+            kwarg,
+            defaults,
+            kw_defaults,
+        })
+    },
+    <params:ParameterListStarArgs<ArgType>> ","? => {
+        let (vararg, kwonlyargs, kw_defaults, kwarg) = params;
+        ast::Arguments {
+            posonlyargs: vec![],
+            args: vec![],
+            kwonlyargs,
+            vararg,
+            kwarg,
+            defaults: vec![],
+            kw_defaults,
+        }
+    },
+    <kwarg:KwargParameter<ArgType>> ","? => {
+        ast::Arguments {
+            posonlyargs: vec![],
+            args: vec![],
+            kwonlyargs: vec![],
+            vararg: None,
+            kwarg,
+            defaults: vec![],
+            kw_defaults: vec![],
+        }
+    },
+};
+
+// Use inline here to make sure the "," is not creating an ambiguity.
+#[inline]
+ParameterDefs<ArgType>: (Vec<(ast::Arg, Option<ast::Expr>)>, Vec<(ast::Arg, Option<ast::Expr>)>) = {
+    <args:OneOrMore<ParameterDef<ArgType>>> => {
+        (vec![], args)
+    },
+    <pos_args:OneOrMore<ParameterDef<ArgType>>> "," "/" <args:("," ParameterDef<ArgType>)*>  => {
+        (pos_args, args.into_iter().map(|e| e.1).collect())
+    },
+};
+
+ParameterDef<ArgType>: (ast::Arg, Option<ast::Expr>) = {
+    <i:ArgType> => (i, None),
+    <i:ArgType> "=" <e:Test> => (i, Some(e)),
+};
+
+UntypedParameter: ast::Arg = {
+    <location:@L> <arg:Identifier> => ast::Arg::new(
+        location,
+        ast::ArgData { arg, annotation: None, type_comment: None },
+    ),
+};
+
+TypedParameter: ast::Arg = {
+    <location:@L> <arg:Identifier> <a:(":" Test)?>=> {
+        let annotation = a.map(|x| Box::new(x.1));
+        ast::Arg::new(location, ast::ArgData { arg, annotation, type_comment: None })
+    },
+};
+
+// Use inline here to make sure the "," is not creating an ambiguity.
+// TODO: figure out another grammar that makes this inline no longer required.
+#[inline]
+ParameterListStarArgs<ArgType>: (Option<Box<ast::Arg>>, Vec<ast::Arg>, Vec<ast::Expr>, Option<Box<ast::Arg>>) = {
+    "*" <va:ArgType?> <kw:("," ParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> => {
+        // Extract keyword arguments:
+        let mut kwonlyargs = Vec::new();
+        let mut kw_defaults = Vec::new();
+        let mut kwargs = Vec::new();
+        for (name, value) in kw.into_iter().map(|x| x.1) {
+            if let Some(value) = value {
+                kwonlyargs.push(name);
+                kw_defaults.push(value);
+            } else {
+                kwargs.push(name);
+            }
+        }
+        kwargs.extend(kwonlyargs.into_iter());
+
+        let kwarg = kwarg.map(|n| n.1).flatten();
+        let va = va.map(Box::new);
+
+        (va, kwargs, kw_defaults, kwarg)
+    }
+};
+
+KwargParameter<ArgType>: Option<Box<ast::Arg>> = {
+    "**" <kwarg:ArgType?> => {
+        kwarg.map(Box::new)
+    }
+};
+
+ClassDef: ast::Stmt = {
+    <decorator_list:Decorator*> <location:@L> "class" <name:Identifier> <a:("(" ArgumentList ")")?> ":" <body:Suite> => {
+        let (bases, keywords) = match a {
+            Some((_, arg, _)) => (arg.args, arg.keywords),
+            None => (vec![], vec![]),
+        };
+        ast::Stmt {
+            custom: (),
+            location,
+            node: ast::StmtKind::ClassDef {
+                name,
+                bases,
+                keywords,
+                body,
+                decorator_list,
+            },
+        }
+    },
+};
+
+// Decorators:
+Decorator: ast::Expr = {
+    <location:@L>"@" <p:NamedExpressionTest> "\n" => {
+        p
+    },
+};
+
+YieldExpr: ast::Expr = {
+    <location:@L> "yield" <value:TestList?> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::Yield { value: value.map(Box::new) }
+    },
+    <location:@L> "yield" "from" <e:Test> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::YieldFrom { value: Box::new(e) }
+    },
+};
+
+Test: ast::Expr = {
+    <expr:OrTest> <condition: (@L "if" OrTest "else" Test)?> => {
+        if let Some(c) = condition {
+            ast::Expr {
+                location: c.0,
+                custom: (),
+                node: ast::ExprKind::IfExp {
+                    test: Box::new(c.2),
+                    body: Box::new(expr),
+                    orelse: Box::new(c.4),
+                }
+            }
+        } else {
+            expr
+        }
+    },
+    LambdaDef,
+};
+
+NamedExpressionTest: ast::Expr = {
+    <location:@L><left: (Identifier ":=")?> <right:Test> => {
+        if let Some(l) = left {
+            ast::Expr {
+                location: location,
+                custom: (),
+                node: ast::ExprKind::NamedExpr {
+                    target: Box::new(ast::Expr::new(
+                        location,
+                        ast::ExprKind::Name { id: l.0, ctx: ast::ExprContext::Store },
+                    )),
+                    value: Box::new(right),
+                }
+            }
+        } else {
+            right
+        }
+    }
+}
+
+LambdaDef: ast::Expr = {
+    <location:@L> "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test> => {
+        let p = p.unwrap_or_else(|| {
+            ast::Arguments {
+                posonlyargs: vec![],
+                args: vec![],
+                vararg: None,
+                kwonlyargs: vec![],
+                kw_defaults: vec![],
+                kwarg: None,
+                defaults: vec![]
+            }
+        });
+        ast::Expr {
+            location,
+            custom: (),
+            node: ast::ExprKind::Lambda {
+                args: Box::new(p),
+                body: Box::new(body)
+            }
+        }
+    }
+}
+
+OrTest: ast::Expr = {
+    <e1:AndTest> <location:@L> <e2:("or" AndTest)*> => {
+        if e2.is_empty() {
+            e1
+        } else {
+            let mut values = vec![e1];
+            values.extend(e2.into_iter().map(|e| e.1));
+            ast::Expr {
+                location,
+                custom: (),
+                node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values }
+            }
+        }
+    },
+};
+
+AndTest: ast::Expr = {
+    <e1:NotTest> <location:@L> <e2:("and" NotTest)*> => {
+        if e2.is_empty() {
+            e1
+        } else {
+            let mut values = vec![e1];
+            values.extend(e2.into_iter().map(|e| e.1));
+            ast::Expr {
+                location,
+                custom: (),
+                node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values }
+            }
+        }
+    },
+};
+
+NotTest: ast::Expr = {
+    <location:@L> "not" <e:NotTest> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::UnaryOp { operand: Box::new(e), op: ast::Unaryop::Not }
+    },
+    Comparison,
+};
+
+Comparison: ast::Expr = {
+    <left:Expression> <location:@L> <comparisons:(CompOp Expression)+> => {
+        let (ops, comparators) = comparisons.into_iter().unzip();
+        ast::Expr {
+            location,
+            custom: (),
+            node: ast::ExprKind::Compare { left: Box::new(left), ops, comparators }
+        }
+    },
+    Expression,
+};
+
+CompOp: ast::Cmpop = {
+    "==" => ast::Cmpop::Eq,
+    "!=" => ast::Cmpop::NotEq,
+    "<" => ast::Cmpop::Lt,
+    "<=" => ast::Cmpop::LtE,
+    ">" => ast::Cmpop::Gt,
+    ">=" => ast::Cmpop::GtE,
+    "in" => ast::Cmpop::In,
+    "not" "in" => ast::Cmpop::NotIn,
+    "is" => ast::Cmpop::Is,
+    "is" "not" => ast::Cmpop::IsNot,
+};
+
+Expression: ast::Expr = {
+    <e1:Expression> <location:@L> "|" <e2:XorExpression> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) }
+    },
+    XorExpression,
+};
+
+XorExpression: ast::Expr = {
+    <e1:XorExpression> <location:@L> "^" <e2:AndExpression> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) }
+    },
+    AndExpression,
+};
+
+AndExpression: ast::Expr = {
+    <e1:AndExpression> <location:@L> "&" <e2:ShiftExpression> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) }
+    },
+    ShiftExpression,
+};
+
+ShiftExpression: ast::Expr = {
+    <e1:ShiftExpression> <location:@L> <op:ShiftOp> <e2:ArithmaticExpression> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) }
+    },
+    ArithmaticExpression,
+};
+
+ShiftOp: ast::Operator = {
+    "<<" => ast::Operator::LShift,
+    ">>" => ast::Operator::RShift,
+};
+
+ArithmaticExpression: ast::Expr = {
+    <a:ArithmaticExpression> <location:@L> <op:AddOp> <b:Term> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
+    },
+    Term,
+};
+
+AddOp: ast::Operator = {
+    "+" => ast::Operator::Add,
+    "-" => ast::Operator::Sub,
+};
+
+Term: ast::Expr = {
+    <a:Term> <location:@L> <op:MulOp> <b:Factor> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
+    },
+    Factor,
+};
+
+MulOp: ast::Operator = {
+    "*" => ast::Operator::Mult,
+    "/" => ast::Operator::Div,
+    "//" => ast::Operator::FloorDiv,
+    "%" => ast::Operator::Mod,
+    "@" => ast::Operator::MatMult,
+};
+
+Factor: ast::Expr = {
+    <location:@L> <op:UnaryOp> <e:Factor> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::UnaryOp { operand: Box::new(e), op }
+    },
+    Power,
+};
+
+UnaryOp: ast::Unaryop = {
+    "+" => ast::Unaryop::UAdd,
+    "-" => ast::Unaryop::USub,
+    "~" => ast::Unaryop::Invert,
+};
+
+Power: ast::Expr = {
+    <e:AtomExpr> <e2:(@L "**" Factor)?> => {
+        match e2 {
+            None => e,
+            Some((location, _, b)) => ast::Expr {
+                location,
+                custom: (),
+                node: ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) }
+            },
+        }
+    }
+};
+
+AtomExpr: ast::Expr = {
+    <location:@L> <is_await:"await"?> <atom:AtomExpr2> => {
+        if is_await.is_some() {
+            ast::Expr {
+                location,
+                custom: (),
+                node: ast::ExprKind::Await { value: Box::new(atom) }
+            }
+        } else {
+            atom
+        }
+    }
+}
+
+AtomExpr2: ast::Expr = {
+    Atom,
+    <f:AtomExpr2> <location:@L> "(" <a:ArgumentList> ")" => {
+        ast::Expr {
+            location,
+            custom: (),
+            node: ast::ExprKind::Call { func: Box::new(f), args: a.args, keywords: a.keywords }
+        }
+    },
+    <e:AtomExpr2> <location:@L> "[" <s:SubscriptList> "]" => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::Subscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load }
+    },
+    <e:AtomExpr2> <location:@L> "." <attr:Identifier> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::Attribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load }
+    },
+};
+
+SubscriptList: ast::Expr = {
+    <location:@L> <s1:Subscript> <s2:("," Subscript)*> <trailing_comma:","?> => {
+        if s2.is_empty() && trailing_comma.is_none() {
+            s1
+        } else {
+            let mut dims = vec![s1];
+            for x in s2 {
+                dims.push(x.1)
+            }
+
+            ast::Expr {
+                location,
+                custom: (),
+                node: ast::ExprKind::Tuple { elts: dims, ctx: ast::ExprContext::Load },
+            }
+        }
+    }
+};
+
+Subscript: ast::Expr = {
+    Test,
+    <e1:Test?> <location:@L> ":" <e2:Test?> <e3:SliceOp?> => {
+        let lower = e1.map(Box::new);
+        let upper = e2.map(Box::new);
+        let step = e3.flatten().map(Box::new);
+        ast::Expr {
+            location,
+            custom: (),
+            node: ast::ExprKind::Slice { lower, upper, step }
+        }
+    }
+};
+
+SliceOp: Option<ast::Expr> = {
+    <location:@L> ":" <e:Test?> => e,
+}
+
+Atom: ast::Expr = {
+    <location:@L> <s:(@L string)+> =>? parse_strings(s).map_err(|e| e.into()),
+    <location:@L> <value:Constant> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::Constant { value, kind: None }
+    },
+    <location:@L> <name:Identifier> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load }
+    },
+    <location:@L> "[" <e:ListLiteralValues?> "]" => {
+        let elts = e.unwrap_or_default();
+        ast::Expr {
+            location,
+            custom: (),
+            node: ast::ExprKind::List { elts, ctx: ast::ExprContext::Load }
+        }
+    },
+    <location:@L> "[" <elt:TestOrStarNamedExpr> <generators:CompFor> "]" => {
+        ast::Expr {
+            location,
+            custom: (),
+            node: ast::ExprKind::ListComp { elt: Box::new(elt), generators }
+        }
+    },
+    <location:@L> "(" <elements:TestOrStarNamedExprList?> ")" =>? {
+        match elements {
+            Some(elt) => {
+                match elt.node {
+                    ast::ExprKind::Starred { .. } => {
+                        Err(LexicalError{
+                            error : LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
+                            location,
+                        }.into())
+                    },
+                    _ => {
+                        Ok(elt)
+                    }
+                }
+            },
+            None => {
+                Ok(ast::Expr {
+                     location,
+                     custom: (),
+                     node: ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load }
+                })
+            }
+        }
+    },
+    "(" <e:YieldExpr> ")" => e,
+    <location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" => {
+        ast::Expr {
+            location,
+            custom: (),
+            node: ast::ExprKind::GeneratorExp { elt: Box::new(elt), generators }
+        }
+    },
+    "(" <location:@L> "**" <e:Expression> ")" =>? {
+        Err(LexicalError{
+            error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()),
+            location,
+        }.into())
+    },
+    <location:@L> "{" <e:DictLiteralValues?> "}" => {
+        let pairs = e.unwrap_or_default();
+
+        let (keys, values) = match pairs.iter().position(|(k,_)| k.is_none()) {
+            Some(unpack_idx) => {
+                let mut pairs = pairs;
+                let (keys, mut values): (_, Vec<_>) = pairs.drain(..unpack_idx).map(|(k, v)| (*k.unwrap(), v)).unzip();
+
+                fn build_map(items: &mut Vec<(ast::Expr, ast::Expr)>) -> ast::Expr {
+                    let location = items[0].0.location;
+                    let (keys, values) = items.drain(..).unzip();
+                    ast::Expr {
+                        location,
+                        custom: (),
+                        node: ast::ExprKind::Dict { keys, values }
+                    }
+                }
+
+                let mut items = Vec::new();
+                for (key, value) in pairs.into_iter() {
+                    if let Some(key) = key {
+                        items.push((*key, value));
+                        continue;
+                    }
+                    if !items.is_empty() {
+                        values.push(build_map(&mut items));
+                    }
+                    values.push(value);
+                }
+                if !items.is_empty() {
+                    values.push(build_map(&mut items));
+                }
+                (keys, values)
+            },
+            None => pairs.into_iter().map(|(k, v)| (*k.unwrap(), v)).unzip()
+        };
+
+        ast::Expr {
+            location,
+            custom: (),
+            node: ast::ExprKind::Dict { keys, values }
+        }
+    },
+    <location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" => {
+        ast::Expr {
+            location,
+            custom: (),
+            node: ast::ExprKind::DictComp {
+                key: Box::new(e1.0),
+                value: Box::new(e1.1),
+                generators,
+            }
+        }
+    },
+    <location:@L> "{" <elts:SetLiteralValues> "}" => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::Set { elts }
+    },
+    <location:@L> "{" <elt:Test> <generators:CompFor> "}" => {
+        ast::Expr {
+            location,
+            custom: (),
+            node: ast::ExprKind::SetComp { elt: Box::new(elt), generators }
+        }
+    },
+    <location:@L> "True" => ast::Expr::new(location, ast::ExprKind::Constant { value: true.into(), kind: None }),
+    <location:@L> "False" => ast::Expr::new(location, ast::ExprKind::Constant { value: false.into(), kind: None }),
+    <location:@L> "None" => ast::Expr::new(location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }),
+    <location:@L> "..." => ast::Expr::new(location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }),
+};
+
+ListLiteralValues: Vec<ast::Expr> = {
+    <e:OneOrMore<TestOrStarNamedExpr>> ","? => e,
+};
+
+DictLiteralValues: Vec<(Option<Box<ast::Expr>>, ast::Expr)> = {
+    <elements:OneOrMore<DictElement>> ","? => elements,
+};
+
+DictEntry: (ast::Expr, ast::Expr) = {
+    <e1: Test> ":" <e2: Test> => (e1, e2),
+};
+
+DictElement: (Option<Box<ast::Expr>>, ast::Expr) = {
+    <e:DictEntry> => (Some(Box::new(e.0)), e.1),
+    "**" <e:Expression> => (None, e),
+};
+
+SetLiteralValues: Vec<ast::Expr> = {
+    <e1:OneOrMore<TestOrStarNamedExpr>> ","? => e1
+};
+
+ExpressionOrStarExpression = {
+    Expression,
+    StarExpr
+};
+
+ExpressionList: ast::Expr = {
+    GenericList<ExpressionOrStarExpression>
+};
+
+ExpressionList2: Vec<ast::Expr> = {
+    <elements:OneOrMore<ExpressionOrStarExpression>> ","? => elements,
+};
+
+// A test list is one of:
+// - a list of expressions
+// - a single expression
+// - a single expression followed by a trailing comma
+#[inline]
+TestList: ast::Expr = {
+    GenericList<TestOrStarExpr>
+};
+
+GenericList<Element>: ast::Expr = {
+    <location:@L> <elts:OneOrMore<Element>> <trailing_comma:","?> => {
+        if elts.len() == 1 && trailing_comma.is_none() {
+            elts.into_iter().next().unwrap()
+        } else {
+            ast::Expr {
+                location,
+                custom: (),
+                node: ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load }
+            }
+        }
+    }
+}
+
+// Test
+StarExpr: ast::Expr = {
+    <location:@L> "*" <e:Expression> => ast::Expr {
+        location,
+        custom: (),
+        node: ast::ExprKind::Starred { value: Box::new(e), ctx: ast::ExprContext::Load },
+    }
+};
+
+// Comprehensions:
+CompFor: Vec<ast::Comprehension> = <c:SingleForComprehension+> => c;
+
+SingleForComprehension: ast::Comprehension = {
+    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest> <ifs:ComprehensionIf*> => {
+        let is_async = is_async.is_some();
+        ast::Comprehension {
+            target: Box::new(target),
+            iter: Box::new(iter),
+            ifs,
+            is_async: if is_async { 1 } else { 0 },
+        }
+    }
+};
+
+ExpressionNoCond: ast::Expr = OrTest;
+ComprehensionIf: ast::Expr = "if" <c:ExpressionNoCond> => c;
+
+ArgumentList: ArgumentList = {
+    <e: Comma<FunctionArgument>> =>? {
+        let arg_list = parse_args(e)?;
+        Ok(arg_list)
+    }
+};
+
+FunctionArgument: (Option<(ast::Location, Option<String>)>, ast::Expr) = {
+    <e:NamedExpressionTest> <c:CompFor?> => {
+        let expr = match c {
+            Some(c) => ast::Expr {
+                location: e.location,
+                custom: (),
+                node: ast::ExprKind::GeneratorExp {
+                    elt: Box::new(e),
+                    generators: c,
+                }
+            },
+            None => e,
+        };
+        (None, expr)
+    },
+    <location:@L> <i:Identifier> "=" <e:Test> => (Some((location, Some(i))), e),
+    <location:@L> "*" <e:Test> => {
+        let expr = ast::Expr::new(
+            location,
+            ast::ExprKind::Starred { value: Box::new(e), ctx: ast::ExprContext::Load },
+        );
+        (None, expr)
+    },
+    <location:@L> "**" <e:Test> => (Some((location, None)), e),
+};
+
+#[inline]
+Comma<T>: Vec<T> = {
+    <items: (<T> ",")*> <last: T?> => {
+        let mut items = items;
+        items.extend(last);
+        items
+    }
+};
+
+#[inline]
+OneOrMore<T>: Vec<T> = {
+    <i1: T> <i2:("," T)*> => {
+        let mut items = vec![i1];
+        items.extend(i2.into_iter().map(|e| e.1));
+        items
+    }
+};
+
+Constant: ast::Constant = {
+    <b:bytes+> => ast::Constant::Bytes(b.into_iter().flatten().collect()),
+    <value:int> => ast::Constant::Int(value),
+    <value:float> => ast::Constant::Float(value),
+    <s:complex> => ast::Constant::Complex { real: s.0, imag: s.1 },
+};
+
+Bytes: Vec<u8> = {
+    <s:bytes+> => {
+        s.into_iter().flatten().collect::<Vec<u8>>()
+    },
+};
+
+Identifier: String = <s:name> => s;
+
+// Hook external lexer:
+extern {
+    type Location = ast::Location;
+    type Error = LexicalError;
+
+    enum lexer::Tok {
+        Indent => lexer::Tok::Indent,
+        Dedent => lexer::Tok::Dedent,
+        StartModule => lexer::Tok::StartModule,
+        StartInteractive => lexer::Tok::StartInteractive,
+        StartExpression => lexer::Tok::StartExpression,
+        "+" => lexer::Tok::Plus,
+        "-" => lexer::Tok::Minus,
+        "~" => lexer::Tok::Tilde,
+        ":" => lexer::Tok::Colon,
+        "." => lexer::Tok::Dot,
+        "..." => lexer::Tok::Ellipsis,
+        "," => lexer::Tok::Comma,
+        "*" => lexer::Tok::Star,
+        "**" => lexer::Tok::DoubleStar,
+        "&" => lexer::Tok::Amper,
+        "@" => lexer::Tok::At,
+        "%" => lexer::Tok::Percent,
+        "//" => lexer::Tok::DoubleSlash,
+        "^" => lexer::Tok::CircumFlex,
+        "|" => lexer::Tok::Vbar,
+        "<<" => lexer::Tok::LeftShift,
+        ">>" => lexer::Tok::RightShift,
+        "/" => lexer::Tok::Slash,
+        "(" => lexer::Tok::Lpar,
+        ")" => lexer::Tok::Rpar,
+        "[" => lexer::Tok::Lsqb,
+        "]" => lexer::Tok::Rsqb,
+        "{" => lexer::Tok::Lbrace,
+        "}" => lexer::Tok::Rbrace,
+        "=" => lexer::Tok::Equal,
+        "+=" => lexer::Tok::PlusEqual,
+        "-=" => lexer::Tok::MinusEqual,
+        "*=" => lexer::Tok::StarEqual,
+        "@=" => lexer::Tok::AtEqual,
+        "/=" => lexer::Tok::SlashEqual,
+        "%=" => lexer::Tok::PercentEqual,
+        "&=" => lexer::Tok::AmperEqual,
+        "|=" => lexer::Tok::VbarEqual,
+        "^=" => lexer::Tok::CircumflexEqual,
+        "<<=" => lexer::Tok::LeftShiftEqual,
+        ">>=" => lexer::Tok::RightShiftEqual,
+        "**=" => lexer::Tok::DoubleStarEqual,
+        "//=" => lexer::Tok::DoubleSlashEqual,
+        ":=" => lexer::Tok::ColonEqual,
+        "==" => lexer::Tok::EqEqual,
+        "!=" => lexer::Tok::NotEqual,
+        "<" => lexer::Tok::Less,
+        "<=" => lexer::Tok::LessEqual,
+        ">" => lexer::Tok::Greater,
+        ">=" => lexer::Tok::GreaterEqual,
+        "->" => lexer::Tok::Rarrow,
+        "and" => lexer::Tok::And,
+        "as" => lexer::Tok::As,
+        "assert" => lexer::Tok::Assert,
+        "async" => lexer::Tok::Async,
+        "await" => lexer::Tok::Await,
+        "break" => lexer::Tok::Break,
+        "class" => lexer::Tok::Class,
+        "continue" => lexer::Tok::Continue,
+        "def" => lexer::Tok::Def,
+        "del" => lexer::Tok::Del,
+        "elif" => lexer::Tok::Elif,
+        "else" => lexer::Tok::Else,
+        "except" => lexer::Tok::Except,
+        "finally" => lexer::Tok::Finally,
+        "for" => lexer::Tok::For,
+        "from" => lexer::Tok::From,
+        "global" => lexer::Tok::Global,
+        "if" => lexer::Tok::If,
+        "in" => lexer::Tok::In,
+        "is" => lexer::Tok::Is,
+        "import" => lexer::Tok::Import,
+        "from" => lexer::Tok::From,
+        "lambda" => lexer::Tok::Lambda,
+        "nonlocal" => lexer::Tok::Nonlocal,
+        "not" => lexer::Tok::Not,
+        "or" => lexer::Tok::Or,
+        "pass" => lexer::Tok::Pass,
+        "raise" => lexer::Tok::Raise,
+        "return" => lexer::Tok::Return,
+        "try" => lexer::Tok::Try,
+        "while" => lexer::Tok::While,
+        "with" => lexer::Tok::With,
+        "yield" => lexer::Tok::Yield,
+        "True" => lexer::Tok::True,
+        "False" => lexer::Tok::False,
+        "None" => lexer::Tok::None,
+        int => lexer::Tok::Int { value: <BigInt> },
+        float => lexer::Tok::Float { value: <f64> },
+        complex => lexer::Tok::Complex { real: <f64>, imag: <f64> },
+        string => lexer::Tok::String { value: <String>, kind: <StringKind> },
+        bytes => lexer::Tok::Bytes { value: <Vec<u8>> },
+        name => lexer::Tok::Name { name: <String> },
+        "\n" => lexer::Tok::Newline,
+        ";" => lexer::Tok::Semi,
+    }
+}
diff --git a/parser/src/error.rs b/parser/src/error.rs
new file mode 100644
index 00000000..cc53ef2f
--- /dev/null
+++ b/parser/src/error.rs
@@ -0,0 +1,235 @@
+//! Define internal parse error types
+//! The goal is to provide a matching and a safe error API, maksing errors from LALR
+use lalrpop_util::ParseError as LalrpopError;
+
+use crate::ast::Location;
+use crate::token::Tok;
+
+use std::error::Error;
+use std::fmt;
+
+/// Represents an error during lexical scanning.
+#[derive(Debug, PartialEq)]
+pub struct LexicalError {
+    pub error: LexicalErrorType,
+    pub location: Location,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum LexicalErrorType {
+    StringError,
+    UnicodeError,
+    NestingError,
+    IndentationError,
+    TabError,
+    TabsAfterSpaces,
+    DefaultArgumentError,
+    PositionalArgumentError,
+    DuplicateKeywordArgumentError,
+    UnrecognizedToken { tok: char },
+    FStringError(FStringErrorType),
+    LineContinuationError,
+    Eof,
+    OtherError(String),
+}
+
+impl fmt::Display for LexicalErrorType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            LexicalErrorType::StringError => write!(f, "Got unexpected string"),
+            LexicalErrorType::FStringError(error) => write!(f, "Got error in f-string: {}", error),
+            LexicalErrorType::UnicodeError => write!(f, "Got unexpected unicode"),
+            LexicalErrorType::NestingError => write!(f, "Got unexpected nesting"),
+            LexicalErrorType::IndentationError => {
+                write!(f, "unindent does not match any outer indentation level")
+            }
+            LexicalErrorType::TabError => {
+                write!(f, "inconsistent use of tabs and spaces in indentation")
+            }
+            LexicalErrorType::TabsAfterSpaces => {
+                write!(f, "Tabs not allowed as part of indentation after spaces")
+            }
+            LexicalErrorType::DefaultArgumentError => {
+                write!(f, "non-default argument follows default argument")
+            }
+            LexicalErrorType::DuplicateKeywordArgumentError => {
+                write!(f, "keyword argument repeated")
+            }
+            LexicalErrorType::PositionalArgumentError => {
+                write!(f, "positional argument follows keyword argument")
+            }
+            LexicalErrorType::UnrecognizedToken { tok } => {
+                write!(f, "Got unexpected token {}", tok)
+            }
+            LexicalErrorType::LineContinuationError => {
+                write!(f, "unexpected character after line continuation character")
+            }
+            LexicalErrorType::Eof => write!(f, "unexpected EOF while parsing"),
+            LexicalErrorType::OtherError(msg) => write!(f, "{}", msg),
+        }
+    }
+}
+
+// TODO: consolidate these with ParseError
+#[derive(Debug, PartialEq)]
+pub struct FStringError {
+    pub error: FStringErrorType,
+    pub location: Location,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum FStringErrorType {
+    UnclosedLbrace,
+    UnopenedRbrace,
+    ExpectedRbrace,
+    InvalidExpression(Box<ParseErrorType>),
+    InvalidConversionFlag,
+    EmptyExpression,
+    MismatchedDelimiter,
+    ExpressionNestedTooDeeply,
+}
+
+impl fmt::Display for FStringErrorType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            FStringErrorType::UnclosedLbrace => write!(f, "Unclosed '{{'"),
+            FStringErrorType::UnopenedRbrace => write!(f, "Unopened '}}'"),
+            FStringErrorType::ExpectedRbrace => write!(f, "Expected '}}' after conversion flag."),
+            FStringErrorType::InvalidExpression(error) => {
+                write!(f, "Invalid expression: {}", error)
+            }
+            FStringErrorType::InvalidConversionFlag => write!(f, "Invalid conversion flag"),
+            FStringErrorType::EmptyExpression => write!(f, "Empty expression"),
+            FStringErrorType::MismatchedDelimiter => write!(f, "Mismatched delimiter"),
+            FStringErrorType::ExpressionNestedTooDeeply => {
+                write!(f, "expressions nested too deeply")
+            }
+        }
+    }
+}
+
+impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
+    fn from(err: FStringError) -> Self {
+        lalrpop_util::ParseError::User {
+            error: LexicalError {
+                error: LexicalErrorType::FStringError(err.error),
+                location: err.location,
+            },
+        }
+    }
+}
+
+/// Represents an error during parsing
+#[derive(Debug, PartialEq)]
+pub struct ParseError {
+    pub error: ParseErrorType,
+    pub location: Location,
+}
+
+#[derive(Debug, PartialEq)]
+pub enum ParseErrorType {
+    /// Parser encountered an unexpected end of input
+    Eof,
+    /// Parser encountered an extra token
+    ExtraToken(Tok),
+    /// Parser encountered an invalid token
+    InvalidToken,
+    /// Parser encountered an unexpected token
+    UnrecognizedToken(Tok, Option<String>),
+    /// Maps to `User` type from `lalrpop-util`
+    Lexical(LexicalErrorType),
+}
+
+/// Convert `lalrpop_util::ParseError` to our internal type
+impl From<LalrpopError<Location, Tok, LexicalError>> for ParseError {
+    fn from(err: LalrpopError<Location, Tok, LexicalError>) -> Self {
+        match err {
+            // TODO: Are there cases where this isn't an EOF?
+            LalrpopError::InvalidToken { location } => ParseError {
+                error: ParseErrorType::Eof,
+                location,
+            },
+            LalrpopError::ExtraToken { token } => ParseError {
+                error: ParseErrorType::ExtraToken(token.1),
+                location: token.0,
+            },
+            LalrpopError::User { error } => ParseError {
+                error: ParseErrorType::Lexical(error.error),
+                location: error.location,
+            },
+            LalrpopError::UnrecognizedToken { token, expected } => {
+                // Hacky, but it's how CPython does it. See PyParser_AddToken,
+                // in particular "Only one possible expected token" comment.
+                let expected = (expected.len() == 1).then(|| expected[0].clone());
+                ParseError {
+                    error: ParseErrorType::UnrecognizedToken(token.1, expected),
+                    location: token.0,
+                }
+            }
+            LalrpopError::UnrecognizedEOF { location, .. } => ParseError {
+                error: ParseErrorType::Eof,
+                location,
+            },
+        }
+    }
+}
+
+impl fmt::Display for ParseError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{} at {}", self.error, self.location)
+    }
+}
+
+impl fmt::Display for ParseErrorType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            ParseErrorType::Eof => write!(f, "Got unexpected EOF"),
+            ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {:?}", tok),
+            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
+            ParseErrorType::UnrecognizedToken(ref tok, ref expected) => {
+                if *tok == Tok::Indent {
+                    write!(f, "unexpected indent")
+                } else if expected.as_deref() == Some("Indent") {
+                    write!(f, "expected an indented block")
+                } else {
+                    write!(f, "invalid syntax. Got unexpected token {}", tok)
+                }
+            }
+            ParseErrorType::Lexical(ref error) => write!(f, "{}", error),
+        }
+    }
+}
+
+impl Error for ParseErrorType {}
+
+impl ParseErrorType {
+    pub fn is_indentation_error(&self) -> bool {
+        match self {
+            ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true,
+            ParseErrorType::UnrecognizedToken(token, expected) => {
+                *token == Tok::Indent || expected.clone() == Some("Indent".to_owned())
+            }
+            _ => false,
+        }
+    }
+    pub fn is_tab_error(&self) -> bool {
+        matches!(
+            self,
+            ParseErrorType::Lexical(LexicalErrorType::TabError)
+                | ParseErrorType::Lexical(LexicalErrorType::TabsAfterSpaces)
+        )
+    }
+}
+
+impl std::ops::Deref for ParseError {
+    type Target = ParseErrorType;
+    fn deref(&self) -> &Self::Target {
+        &self.error
+    }
+}
+
+impl Error for ParseError {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        None
+    }
+}
diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
new file mode 100644
index 00000000..28693061
--- /dev/null
+++ b/parser/src/fstring.rs
@@ -0,0 +1,427 @@
+use self::FStringErrorType::*;
+use crate::{
+    ast::{Constant, ConversionFlag, Expr, ExprKind, Location},
+    error::{FStringError, FStringErrorType, ParseError},
+    parser::parse_expression,
+};
+use std::{iter, mem, str};
+
+struct FStringParser<'a> {
+    chars: iter::Peekable<str::Chars<'a>>,
+    str_location: Location,
+    recurse_lvl: u8,
+}
+
+impl<'a> FStringParser<'a> {
+    fn new(source: &'a str, str_location: Location, recurse_lvl: u8) -> Self {
+        Self {
+            chars: source.chars().peekable(),
+            str_location,
+            recurse_lvl,
+        }
+    }
+
+    #[inline]
+    fn expr(&self, node: ExprKind) -> Expr {
+        Expr::new(self.str_location, node)
+    }
+
+    fn parse_formatted_value(&mut self) -> Result<Vec<Expr>, FStringErrorType> {
+        let mut expression = String::new();
+        let mut spec = None;
+        let mut delims = Vec::new();
+        let mut conversion = ConversionFlag::None;
+        let mut self_documenting = false;
+        let mut trailing_seq = String::new();
+
+        while let Some(ch) = self.chars.next() {
+            match ch {
+                // can be integrated better with the remainign code, but as a starting point ok
+                // in general I would do here a tokenizing of the fstrings to omit this peeking.
+                '!' if self.chars.peek() == Some(&'=') => {
+                    expression.push_str("!=");
+                    self.chars.next();
+                }
+
+                '=' if self.chars.peek() == Some(&'=') => {
+                    expression.push_str("==");
+                    self.chars.next();
+                }
+
+                '>' if self.chars.peek() == Some(&'=') => {
+                    expression.push_str(">=");
+                    self.chars.next();
+                }
+
+                '<' if self.chars.peek() == Some(&'=') => {
+                    expression.push_str("<=");
+                    self.chars.next();
+                }
+
+                '!' if delims.is_empty() && self.chars.peek() != Some(&'=') => {
+                    if expression.trim().is_empty() {
+                        return Err(EmptyExpression);
+                    }
+
+                    conversion = match self.chars.next() {
+                        Some('s') => ConversionFlag::Str,
+                        Some('a') => ConversionFlag::Ascii,
+                        Some('r') => ConversionFlag::Repr,
+                        Some(_) => {
+                            return Err(InvalidConversionFlag);
+                        }
+                        None => {
+                            return Err(ExpectedRbrace);
+                        }
+                    };
+
+                    if let Some(&peek) = self.chars.peek() {
+                        if peek != '}' && peek != ':' {
+                            return Err(ExpectedRbrace);
+                        }
+                    } else {
+                        return Err(ExpectedRbrace);
+                    }
+                }
+
+                // match a python 3.8 self documenting expression
+                // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}'
+                '=' if self.chars.peek() != Some(&'=') && delims.is_empty() => {
+                    self_documenting = true;
+                }
+
+                ':' if delims.is_empty() => {
+                    let mut nested = 0;
+                    let mut spec_constructor = Vec::new();
+                    let mut constant_piece = String::new();
+                    let mut formatted_value_piece = String::new();
+                    while let Some(&next) = self.chars.peek() {
+                        match next {
+                            '{' if nested > 0 => {
+                                nested += 1;
+                                formatted_value_piece.push(next);
+                            }
+                            '}' if nested > 0 => {
+                                nested -= 1;
+                                if nested == 0 {
+                                    formatted_value_piece.push(next);
+                                    spec_constructor.push(
+                                        self.expr(ExprKind::FormattedValue {
+                                            value: Box::new(
+                                                FStringParser::new(
+                                                    &formatted_value_piece,
+                                                    Location::default(),
+                                                    &self.recurse_lvl + 1,
+                                                )
+                                                .parse()?,
+                                            ),
+                                            conversion: ConversionFlag::None as _,
+                                            format_spec: None,
+                                        }),
+                                    );
+                                    formatted_value_piece.clear();
+                                } else {
+                                    formatted_value_piece.push(next);
+                                }
+                            }
+                            _ if nested > 0 => {
+                                formatted_value_piece.push(next);
+                            }
+                            '{' => {
+                                nested += 1;
+                                spec_constructor.push(self.expr(ExprKind::Constant {
+                                    value: constant_piece.to_owned().into(),
+                                    kind: None,
+                                }));
+                                constant_piece.clear();
+                                formatted_value_piece.push(next);
+                                formatted_value_piece.push(' ');
+                            }
+                            '}' => break,
+                            _ => {
+                                constant_piece.push(next);
+                            }
+                        }
+                        self.chars.next();
+                    }
+                    spec_constructor.push(self.expr(ExprKind::Constant {
+                        value: constant_piece.to_owned().into(),
+                        kind: None,
+                    }));
+                    constant_piece.clear();
+                    if nested > 0 {
+                        return Err(UnclosedLbrace);
+                    }
+                    spec = Some(Box::new(self.expr(ExprKind::JoinedStr {
+                        values: spec_constructor,
+                    })))
+                }
+                '(' | '{' | '[' => {
+                    expression.push(ch);
+                    delims.push(ch);
+                }
+                ')' => {
+                    if delims.pop() != Some('(') {
+                        return Err(MismatchedDelimiter);
+                    }
+                    expression.push(ch);
+                }
+                ']' => {
+                    if delims.pop() != Some('[') {
+                        return Err(MismatchedDelimiter);
+                    }
+                    expression.push(ch);
+                }
+                '}' if !delims.is_empty() => {
+                    if delims.pop() != Some('{') {
+                        return Err(MismatchedDelimiter);
+                    }
+                    expression.push(ch);
+                }
+                '}' => {
+                    if expression.is_empty() {
+                        return Err(EmptyExpression);
+                    }
+                    let ret = if !self_documenting {
+                        vec![self.expr(ExprKind::FormattedValue {
+                            value: Box::new(
+                                parse_fstring_expr(&expression)
+                                    .map_err(|e| InvalidExpression(Box::new(e.error)))?,
+                            ),
+                            conversion: conversion as _,
+                            format_spec: spec,
+                        })]
+                    } else {
+                        vec![
+                            self.expr(ExprKind::Constant {
+                                value: Constant::Str(expression.clone() + "="),
+                                kind: None,
+                            }),
+                            self.expr(ExprKind::Constant {
+                                value: trailing_seq.into(),
+                                kind: None,
+                            }),
+                            self.expr(ExprKind::FormattedValue {
+                                value: Box::new(
+                                    parse_fstring_expr(&expression)
+                                        .map_err(|e| InvalidExpression(Box::new(e.error)))?,
+                                ),
+                                conversion: (if conversion == ConversionFlag::None && spec.is_none()
+                                {
+                                    ConversionFlag::Repr
+                                } else {
+                                    conversion
+                                }) as _,
+                                format_spec: spec,
+                            }),
+                        ]
+                    };
+                    return Ok(ret);
+                }
+                '"' | '\'' => {
+                    expression.push(ch);
+                    for next in &mut self.chars {
+                        expression.push(next);
+                        if next == ch {
+                            break;
+                        }
+                    }
+                }
+                ' ' if self_documenting => {
+                    trailing_seq.push(ch);
+                }
+                _ => {
+                    if self_documenting {
+                        return Err(ExpectedRbrace);
+                    }
+                    expression.push(ch);
+                }
+            }
+        }
+        Err(UnclosedLbrace)
+    }
+
+    fn parse(mut self) -> Result<Expr, FStringErrorType> {
+        if self.recurse_lvl >= 2 {
+            return Err(ExpressionNestedTooDeeply);
+        }
+
+        let mut content = String::new();
+        let mut values = vec![];
+
+        while let Some(ch) = self.chars.next() {
+            match ch {
+                '{' => {
+                    if let Some('{') = self.chars.peek() {
+                        self.chars.next();
+                        content.push('{');
+                    } else {
+                        if !content.is_empty() {
+                            values.push(self.expr(ExprKind::Constant {
+                                value: mem::take(&mut content).into(),
+                                kind: None,
+                            }));
+                        }
+
+                        values.extend(self.parse_formatted_value()?);
+                    }
+                }
+                '}' => {
+                    if let Some('}') = self.chars.peek() {
+                        self.chars.next();
+                        content.push('}');
+                    } else {
+                        return Err(UnopenedRbrace);
+                    }
+                }
+                _ => {
+                    content.push(ch);
+                }
+            }
+        }
+
+        if !content.is_empty() {
+            values.push(self.expr(ExprKind::Constant {
+                value: content.into(),
+                kind: None,
+            }))
+        }
+
+        Ok(self.expr(ExprKind::JoinedStr { values }))
+    }
+}
+
+fn parse_fstring_expr(source: &str) -> Result<Expr, ParseError> {
+    let fstring_body = format!("({})", source);
+    parse_expression(&fstring_body)
+}
+
+/// Parse an fstring from a string, located at a certain position in the sourcecode.
+/// In case of errors, we will get the location and the error returned.
+pub fn parse_located_fstring(source: &str, location: Location) -> Result<Expr, FStringError> {
+    FStringParser::new(source, location, 0)
+        .parse()
+        .map_err(|error| FStringError { error, location })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn parse_fstring(source: &str) -> Result<Expr, FStringErrorType> {
+        FStringParser::new(source, Location::default(), 0).parse()
+    }
+
+    #[test]
+    fn test_parse_fstring() {
+        let source = "{a}{ b }{{foo}}";
+        let parse_ast = parse_fstring(source).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_nested_spec() {
+        let source = "{foo:{spec}}";
+        let parse_ast = parse_fstring(source).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_not_nested_spec() {
+        let source = "{foo:spec}";
+        let parse_ast = parse_fstring(source).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_empty_fstring() {
+        insta::assert_debug_snapshot!(parse_fstring("").unwrap());
+    }
+
+    #[test]
+    fn test_fstring_parse_selfdocumenting_base() {
+        let src = "{user=}";
+        let parse_ast = parse_fstring(src).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_fstring_parse_selfdocumenting_base_more() {
+        let src = "mix {user=} with text and {second=}";
+        let parse_ast = parse_fstring(src).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_fstring_parse_selfdocumenting_format() {
+        let src = "{user=:>10}";
+        let parse_ast = parse_fstring(src).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_invalid_fstring() {
+        assert_eq!(parse_fstring("{5!a"), Err(ExpectedRbrace));
+        assert_eq!(parse_fstring("{5!a1}"), Err(ExpectedRbrace));
+        assert_eq!(parse_fstring("{5!"), Err(ExpectedRbrace));
+        assert_eq!(parse_fstring("abc{!a 'cat'}"), Err(EmptyExpression));
+        assert_eq!(parse_fstring("{!a"), Err(EmptyExpression));
+        assert_eq!(parse_fstring("{ !a}"), Err(EmptyExpression));
+
+        assert_eq!(parse_fstring("{5!}"), Err(InvalidConversionFlag));
+        assert_eq!(parse_fstring("{5!x}"), Err(InvalidConversionFlag));
+
+        assert_eq!(parse_fstring("{a:{a:{b}}}"), Err(ExpressionNestedTooDeeply));
+
+        assert_eq!(parse_fstring("{a:b}}"), Err(UnopenedRbrace));
+        assert_eq!(parse_fstring("}"), Err(UnopenedRbrace));
+        assert_eq!(parse_fstring("{a:{b}"), Err(UnclosedLbrace));
+        assert_eq!(parse_fstring("{"), Err(UnclosedLbrace));
+
+        assert_eq!(parse_fstring("{}"), Err(EmptyExpression));
+
+        // TODO: check for InvalidExpression enum?
+        assert!(parse_fstring("{class}").is_err());
+    }
+
+    #[test]
+    fn test_parse_fstring_not_equals() {
+        let source = "{1 != 2}";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_equals() {
+        let source = "{42 == 42}";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_selfdoc_prec_space() {
+        let source = "{x   =}";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_selfdoc_trailing_space() {
+        let source = "{x=   }";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_yield_expr() {
+        let source = "{yield}";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+}
diff --git a/parser/src/function.rs b/parser/src/function.rs
new file mode 100644
index 00000000..68d890f2
--- /dev/null
+++ b/parser/src/function.rs
@@ -0,0 +1,96 @@
+use ahash::RandomState;
+use std::collections::HashSet;
+
+use crate::ast;
+use crate::error::{LexicalError, LexicalErrorType};
+
+pub struct ArgumentList {
+    pub args: Vec<ast::Expr>,
+    pub keywords: Vec<ast::Keyword>,
+}
+
+type ParameterDefs = (Vec<ast::Arg>, Vec<ast::Arg>, Vec<ast::Expr>);
+type ParameterDef = (ast::Arg, Option<ast::Expr>);
+
+pub fn parse_params(
+    params: (Vec<ParameterDef>, Vec<ParameterDef>),
+) -> Result<ParameterDefs, LexicalError> {
+    let mut posonly = Vec::with_capacity(params.0.len());
+    let mut names = Vec::with_capacity(params.1.len());
+    let mut defaults = vec![];
+
+    let mut try_default = |name: &ast::Arg, default| {
+        if let Some(default) = default {
+            defaults.push(default);
+        } else if !defaults.is_empty() {
+            // Once we have started with defaults, all remaining arguments must
+            // have defaults
+            return Err(LexicalError {
+                error: LexicalErrorType::DefaultArgumentError,
+                location: name.location,
+            });
+        }
+        Ok(())
+    };
+
+    for (name, default) in params.0 {
+        try_default(&name, default)?;
+        posonly.push(name);
+    }
+
+    for (name, default) in params.1 {
+        try_default(&name, default)?;
+        names.push(name);
+    }
+
+    Ok((posonly, names, defaults))
+}
+
+type FunctionArgument = (Option<(ast::Location, Option<String>)>, ast::Expr);
+
+pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, LexicalError> {
+    let mut args = vec![];
+    let mut keywords = vec![];
+
+    let mut keyword_names = HashSet::with_capacity_and_hasher(func_args.len(), RandomState::new());
+    for (name, value) in func_args {
+        match name {
+            Some((location, name)) => {
+                if let Some(keyword_name) = &name {
+                    if keyword_names.contains(keyword_name) {
+                        return Err(LexicalError {
+                            error: LexicalErrorType::DuplicateKeywordArgumentError,
+                            location,
+                        });
+                    }
+
+                    keyword_names.insert(keyword_name.clone());
+                }
+
+                keywords.push(ast::Keyword::new(
+                    location,
+                    ast::KeywordData {
+                        arg: name,
+                        value: Box::new(value),
+                    },
+                ));
+            }
+            None => {
+                // Allow starred args after keyword arguments.
+                if !keywords.is_empty() && !is_starred(&value) {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::PositionalArgumentError,
+                        location: value.location,
+                    });
+                }
+
+                args.push(value);
+            }
+        }
+    }
+    Ok(ArgumentList { args, keywords })
+}
+
+fn is_starred(exp: &ast::Expr) -> bool {
+    matches!(exp.node, ast::ExprKind::Starred { .. })
+}
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
new file mode 100644
index 00000000..59c76a59
--- /dev/null
+++ b/parser/src/lexer.rs
@@ -0,0 +1,1712 @@
+//! This module takes care of lexing python source text.
+//!
+//! This means source code is translated into separate tokens.
+
+use super::token::StringKind;
+pub use super::token::Tok;
+use crate::ast::Location;
+use crate::error::{LexicalError, LexicalErrorType};
+use num_bigint::BigInt;
+use num_traits::identities::Zero;
+use num_traits::Num;
+use std::char;
+use std::cmp::Ordering;
+use std::str::FromStr;
+use unic_emoji_char::is_emoji_presentation;
+use unic_ucd_ident::{is_xid_continue, is_xid_start};
+
+#[derive(Clone, Copy, PartialEq, Debug, Default)]
+struct IndentationLevel {
+    tabs: usize,
+    spaces: usize,
+}
+
+impl IndentationLevel {
+    fn compare_strict(
+        &self,
+        other: &IndentationLevel,
+        location: Location,
+    ) -> Result<Ordering, LexicalError> {
+        // We only know for sure that we're smaller or bigger if tabs
+        // and spaces both differ in the same direction. Otherwise we're
+        // dependent on the size of tabs.
+        match self.tabs.cmp(&other.tabs) {
+            Ordering::Less => {
+                if self.spaces <= other.spaces {
+                    Ok(Ordering::Less)
+                } else {
+                    Err(LexicalError {
+                        location,
+                        error: LexicalErrorType::TabError,
+                    })
+                }
+            }
+            Ordering::Greater => {
+                if self.spaces >= other.spaces {
+                    Ok(Ordering::Greater)
+                } else {
+                    Err(LexicalError {
+                        location,
+                        error: LexicalErrorType::TabError,
+                    })
+                }
+            }
+            Ordering::Equal => Ok(self.spaces.cmp(&other.spaces)),
+        }
+    }
+}
+
+pub struct Lexer<T: Iterator<Item = char>> {
+    chars: T,
+    at_begin_of_line: bool,
+    nesting: usize, // Amount of parenthesis
+    indentation_stack: Vec<IndentationLevel>,
+    pending: Vec<Spanned>,
+    chr0: Option<char>,
+    chr1: Option<char>,
+    chr2: Option<char>,
+    location: Location,
+}
+
+// generated in build.rs, in gen_phf()
+pub static KEYWORDS: phf::Map<&'static str, Tok> =
+    include!(concat!(env!("OUT_DIR"), "/keywords.rs"));
+
+pub type Spanned = (Location, Tok, Location);
+pub type LexResult = Result<Spanned, LexicalError>;
+
+#[inline]
+pub fn make_tokenizer(source: &str) -> impl Iterator<Item = LexResult> + '_ {
+    make_tokenizer_located(source, Location::new(0, 0))
+}
+
+pub fn make_tokenizer_located(
+    source: &str,
+    start_location: Location,
+) -> impl Iterator<Item = LexResult> + '_ {
+    let nlh = NewlineHandler::new(source.chars());
+    Lexer::new(nlh, start_location)
+}
+
+// The newline handler is an iterator which collapses different newline
+// types into \n always.
+pub struct NewlineHandler<T: Iterator<Item = char>> {
+    source: T,
+    chr0: Option<char>,
+    chr1: Option<char>,
+}
+
+impl<T> NewlineHandler<T>
+where
+    T: Iterator<Item = char>,
+{
+    pub fn new(source: T) -> Self {
+        let mut nlh = NewlineHandler {
+            source,
+            chr0: None,
+            chr1: None,
+        };
+        nlh.shift();
+        nlh.shift();
+        nlh
+    }
+
+    fn shift(&mut self) -> Option<char> {
+        let result = self.chr0;
+        self.chr0 = self.chr1;
+        self.chr1 = self.source.next();
+        result
+    }
+}
+
+impl<T> Iterator for NewlineHandler<T>
+where
+    T: Iterator<Item = char>,
+{
+    type Item = char;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // Collapse \r\n into \n
+        loop {
+            if self.chr0 == Some('\r') {
+                if self.chr1 == Some('\n') {
+                    // Transform windows EOL into \n
+                    self.shift();
+                } else {
+                    // Transform MAC EOL into \n
+                    self.chr0 = Some('\n')
+                }
+            } else {
+                break;
+            }
+        }
+
+        self.shift()
+    }
+}
+
+/// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
+const MAX_UNICODE_NAME: usize = 88;
+
+impl<T> Lexer<T>
+where
+    T: Iterator<Item = char>,
+{
+    pub fn new(input: T, start: Location) -> Self {
+        let mut lxr = Lexer {
+            chars: input,
+            at_begin_of_line: true,
+            nesting: 0,
+            indentation_stack: vec![Default::default()],
+            pending: Vec::new(),
+            chr0: None,
+            location: start,
+            chr1: None,
+            chr2: None,
+        };
+        lxr.next_char();
+        lxr.next_char();
+        lxr.next_char();
+        // Start at top row (=1) left column (=1)
+        lxr.location.reset();
+        lxr
+    }
+
+    // Lexer helper functions:
+    fn lex_identifier(&mut self) -> LexResult {
+        let mut name = String::new();
+        let start_pos = self.get_pos();
+
+        // Detect potential string like rb'' b'' f'' u'' r''
+        let mut saw_b = false;
+        let mut saw_r = false;
+        let mut saw_u = false;
+        let mut saw_f = false;
+        loop {
+            // Detect r"", f"", b"" and u""
+            if !(saw_b || saw_u || saw_f) && matches!(self.chr0, Some('b') | Some('B')) {
+                saw_b = true;
+            } else if !(saw_b || saw_r || saw_u || saw_f)
+                && matches!(self.chr0, Some('u') | Some('U'))
+            {
+                saw_u = true;
+            } else if !(saw_r || saw_u) && (self.chr0 == Some('r') || self.chr0 == Some('R')) {
+                saw_r = true;
+            } else if !(saw_b || saw_u || saw_f)
+                && (self.chr0 == Some('f') || self.chr0 == Some('F'))
+            {
+                saw_f = true;
+            } else {
+                break;
+            }
+
+            // Take up char into name:
+            name.push(self.next_char().unwrap());
+
+            // Check if we have a string:
+            if self.chr0 == Some('"') || self.chr0 == Some('\'') {
+                return self.lex_string(saw_b, saw_r, saw_u, saw_f);
+            }
+        }
+
+        while self.is_identifier_continuation() {
+            name.push(self.next_char().unwrap());
+        }
+        let end_pos = self.get_pos();
+
+        if let Some(tok) = KEYWORDS.get(name.as_str()) {
+            Ok((start_pos, tok.clone(), end_pos))
+        } else {
+            Ok((start_pos, Tok::Name { name }, end_pos))
+        }
+    }
+
+    /// Numeric lexing. The feast can start!
+    fn lex_number(&mut self) -> LexResult {
+        let start_pos = self.get_pos();
+        if self.chr0 == Some('0') {
+            if self.chr1 == Some('x') || self.chr1 == Some('X') {
+                // Hex!
+                self.next_char();
+                self.next_char();
+                self.lex_number_radix(start_pos, 16)
+            } else if self.chr1 == Some('o') || self.chr1 == Some('O') {
+                // Octal style!
+                self.next_char();
+                self.next_char();
+                self.lex_number_radix(start_pos, 8)
+            } else if self.chr1 == Some('b') || self.chr1 == Some('B') {
+                // Binary!
+                self.next_char();
+                self.next_char();
+                self.lex_number_radix(start_pos, 2)
+            } else {
+                self.lex_normal_number()
+            }
+        } else {
+            self.lex_normal_number()
+        }
+    }
+
+    /// Lex a hex/octal/decimal/binary number without a decimal point.
+    fn lex_number_radix(&mut self, start_pos: Location, radix: u32) -> LexResult {
+        let value_text = self.radix_run(radix);
+        let end_pos = self.get_pos();
+        let value = BigInt::from_str_radix(&value_text, radix).map_err(|e| LexicalError {
+            error: LexicalErrorType::OtherError(format!("{:?}", e)),
+            location: start_pos,
+        })?;
+        Ok((start_pos, Tok::Int { value }, end_pos))
+    }
+
+    /// Lex a normal number, that is, no octal, hex or binary number.
+    fn lex_normal_number(&mut self) -> LexResult {
+        let start_pos = self.get_pos();
+        let start_is_zero = self.chr0 == Some('0');
+        // Normal number:
+        let mut value_text = self.radix_run(10);
+
+        // If float:
+        if self.chr0 == Some('.') || self.at_exponent() {
+            // Take '.':
+            if self.chr0 == Some('.') {
+                if self.chr1 == Some('_') {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),
+                        location: self.get_pos(),
+                    });
+                }
+                value_text.push(self.next_char().unwrap());
+                value_text.push_str(&self.radix_run(10));
+            }
+
+            // 1e6 for example:
+            if self.chr0 == Some('e') || self.chr0 == Some('E') {
+                if self.chr1 == Some('_') {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),
+                        location: self.get_pos(),
+                    });
+                }
+                value_text.push(self.next_char().unwrap().to_ascii_lowercase());
+                // Optional +/-
+                if self.chr0 == Some('-') || self.chr0 == Some('+') {
+                    if self.chr1 == Some('_') {
+                        return Err(LexicalError {
+                            error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),
+                            location: self.get_pos(),
+                        });
+                    }
+                    value_text.push(self.next_char().unwrap());
+                }
+
+                value_text.push_str(&self.radix_run(10));
+            }
+
+            let value = f64::from_str(&value_text).map_err(|_| LexicalError {
+                error: LexicalErrorType::OtherError("Invalid decimal literal".to_owned()),
+                location: self.get_pos(),
+            })?;
+            // Parse trailing 'j':
+            if self.chr0 == Some('j') || self.chr0 == Some('J') {
+                self.next_char();
+                let end_pos = self.get_pos();
+                Ok((
+                    start_pos,
+                    Tok::Complex {
+                        real: 0.0,
+                        imag: value,
+                    },
+                    end_pos,
+                ))
+            } else {
+                let end_pos = self.get_pos();
+                Ok((start_pos, Tok::Float { value }, end_pos))
+            }
+        } else {
+            // Parse trailing 'j':
+            if self.chr0 == Some('j') || self.chr0 == Some('J') {
+                self.next_char();
+                let end_pos = self.get_pos();
+                let imag = f64::from_str(&value_text).unwrap();
+                Ok((start_pos, Tok::Complex { real: 0.0, imag }, end_pos))
+            } else {
+                let end_pos = self.get_pos();
+                let value = value_text.parse::<BigInt>().unwrap();
+                if start_is_zero && !value.is_zero() {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::OtherError("Invalid Token".to_owned()),
+                        location: self.get_pos(),
+                    });
+                }
+                Ok((start_pos, Tok::Int { value }, end_pos))
+            }
+        }
+    }
+
+    /// Consume a sequence of numbers with the given radix,
+    /// the digits can be decorated with underscores
+    /// like this: '1_2_3_4' == '1234'
+    fn radix_run(&mut self, radix: u32) -> String {
+        let mut value_text = String::new();
+
+        loop {
+            if let Some(c) = self.take_number(radix) {
+                value_text.push(c);
+            } else if self.chr0 == Some('_') && Lexer::<T>::is_digit_of_radix(self.chr1, radix) {
+                self.next_char();
+            } else {
+                break;
+            }
+        }
+        value_text
+    }
+
+    /// Consume a single character with the given radix.
+    fn take_number(&mut self, radix: u32) -> Option<char> {
+        let take_char = Lexer::<T>::is_digit_of_radix(self.chr0, radix);
+
+        if take_char {
+            Some(self.next_char().unwrap())
+        } else {
+            None
+        }
+    }
+
+    /// Test if a digit is of a certain radix.
+    fn is_digit_of_radix(c: Option<char>, radix: u32) -> bool {
+        match radix {
+            2 => matches!(c, Some('0'..='1')),
+            8 => matches!(c, Some('0'..='7')),
+            10 => matches!(c, Some('0'..='9')),
+            16 => matches!(c, Some('0'..='9') | Some('a'..='f') | Some('A'..='F')),
+            other => unimplemented!("Radix not implemented: {}", other),
+        }
+    }
+
+    /// Test if we face '[eE][-+]?[0-9]+'
+    fn at_exponent(&self) -> bool {
+        match self.chr0 {
+            Some('e') | Some('E') => match self.chr1 {
+                Some('+') | Some('-') => matches!(self.chr2, Some('0'..='9')),
+                Some('0'..='9') => true,
+                _ => false,
+            },
+            _ => false,
+        }
+    }
+
+    /// Skip everything until end of line
+    fn lex_comment(&mut self) {
+        self.next_char();
+        loop {
+            match self.chr0 {
+                Some('\n') => return,
+                Some(_) => {}
+                None => return,
+            }
+            self.next_char();
+        }
+    }
+
+    fn unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
+        let mut p: u32 = 0u32;
+        let unicode_error = LexicalError {
+            error: LexicalErrorType::UnicodeError,
+            location: self.get_pos(),
+        };
+        for i in 1..=literal_number {
+            match self.next_char() {
+                Some(c) => match c.to_digit(16) {
+                    Some(d) => p += d << ((literal_number - i) * 4),
+                    None => return Err(unicode_error),
+                },
+                None => return Err(unicode_error),
+            }
+        }
+        match p {
+            0xD800..=0xDFFF => Ok(std::char::REPLACEMENT_CHARACTER),
+            _ => std::char::from_u32(p).ok_or(unicode_error),
+        }
+    }
+
+    fn parse_octet(&mut self, first: char) -> char {
+        let mut octet_content = String::new();
+        octet_content.push(first);
+        while octet_content.len() < 3 {
+            if let Some('0'..='7') = self.chr0 {
+                octet_content.push(self.next_char().unwrap())
+            } else {
+                break;
+            }
+        }
+        let value = u32::from_str_radix(&octet_content, 8).unwrap();
+        char::from_u32(value).unwrap()
+    }
+
+    fn parse_unicode_name(&mut self) -> Result<char, LexicalError> {
+        let start_pos = self.get_pos();
+        match self.next_char() {
+            Some('{') => {}
+            _ => {
+                return Err(LexicalError {
+                    error: LexicalErrorType::StringError,
+                    location: start_pos,
+                })
+            }
+        }
+        let start_pos = self.get_pos();
+        let mut name = String::new();
+        loop {
+            match self.next_char() {
+                Some('}') => break,
+                Some(c) => name.push(c),
+                None => {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::StringError,
+                        location: self.get_pos(),
+                    })
+                }
+            }
+        }
+
+        if name.len() > MAX_UNICODE_NAME {
+            return Err(LexicalError {
+                error: LexicalErrorType::UnicodeError,
+                location: self.get_pos(),
+            });
+        }
+
+        unicode_names2::character(&name).ok_or(LexicalError {
+            error: LexicalErrorType::UnicodeError,
+            location: start_pos,
+        })
+    }
+
+    fn lex_string(
+        &mut self,
+        is_bytes: bool,
+        is_raw: bool,
+        is_unicode: bool,
+        is_fstring: bool,
+    ) -> LexResult {
+        let quote_char = self.next_char().unwrap();
+        let mut string_content = String::new();
+        let start_pos = self.get_pos();
+
+        // If the next two characters are also the quote character, then we have a triple-quoted
+        // string; consume those two characters and ensure that we require a triple-quote to close
+        let triple_quoted = if self.chr0 == Some(quote_char) && self.chr1 == Some(quote_char) {
+            self.next_char();
+            self.next_char();
+            true
+        } else {
+            false
+        };
+
+        loop {
+            match self.next_char() {
+                Some('\\') => {
+                    if self.chr0 == Some(quote_char) && !is_raw {
+                        string_content.push(quote_char);
+                        self.next_char();
+                    } else if is_raw {
+                        string_content.push('\\');
+                        if let Some(c) = self.next_char() {
+                            string_content.push(c)
+                        } else {
+                            return Err(LexicalError {
+                                error: LexicalErrorType::StringError,
+                                location: self.get_pos(),
+                            });
+                        }
+                    } else {
+                        match self.next_char() {
+                            Some('\\') => {
+                                string_content.push('\\');
+                            }
+                            Some('\'') => string_content.push('\''),
+                            Some('\"') => string_content.push('\"'),
+                            Some('\n') => {
+                                // Ignore Unix EOL character
+                            }
+                            Some('a') => string_content.push('\x07'),
+                            Some('b') => string_content.push('\x08'),
+                            Some('f') => string_content.push('\x0c'),
+                            Some('n') => {
+                                string_content.push('\n');
+                            }
+                            Some('r') => string_content.push('\r'),
+                            Some('t') => {
+                                string_content.push('\t');
+                            }
+                            Some('v') => string_content.push('\x0b'),
+                            Some(o @ '0'..='7') => string_content.push(self.parse_octet(o)),
+                            Some('x') => string_content.push(self.unicode_literal(2)?),
+                            Some('u') if !is_bytes => string_content.push(self.unicode_literal(4)?),
+                            Some('U') if !is_bytes => string_content.push(self.unicode_literal(8)?),
+                            Some('N') if !is_bytes => {
+                                string_content.push(self.parse_unicode_name()?)
+                            }
+                            Some(c) => {
+                                string_content.push('\\');
+                                string_content.push(c);
+                            }
+                            None => {
+                                return Err(LexicalError {
+                                    error: LexicalErrorType::StringError,
+                                    location: self.get_pos(),
+                                });
+                            }
+                        }
+                    }
+                }
+                Some(c) => {
+                    if c == quote_char {
+                        if triple_quoted {
+                            // Look ahead at the next two characters; if we have two more
+                            // quote_chars, it's the end of the string; consume the remaining
+                            // closing quotes and break the loop
+                            if self.chr0 == Some(quote_char) && self.chr1 == Some(quote_char) {
+                                self.next_char();
+                                self.next_char();
+                                break;
+                            }
+                            string_content.push(c);
+                        } else {
+                            break;
+                        }
+                    } else {
+                        if (c == '\n' && !triple_quoted) || (is_bytes && !c.is_ascii()) {
+                            return Err(LexicalError {
+                                error: LexicalErrorType::Eof,
+                                location: self.get_pos(),
+                            });
+                        }
+                        string_content.push(c);
+                    }
+                }
+                None => {
+                    return Err(LexicalError {
+                        error: if triple_quoted {
+                            LexicalErrorType::Eof
+                        } else {
+                            LexicalErrorType::StringError
+                        },
+                        location: self.get_pos(),
+                    });
+                }
+            }
+        }
+        let end_pos = self.get_pos();
+
+        let tok = if is_bytes {
+            Tok::Bytes {
+                value: string_content.chars().map(|c| c as u8).collect(),
+            }
+        } else {
+            let kind = if is_fstring {
+                StringKind::F
+            } else if is_unicode {
+                StringKind::U
+            } else {
+                StringKind::Normal
+            };
+            Tok::String {
+                value: string_content,
+                kind,
+            }
+        };
+
+        Ok((start_pos, tok, end_pos))
+    }
+
+    fn is_identifier_start(&self, c: char) -> bool {
+        c == '_' || is_xid_start(c)
+    }
+
+    fn is_identifier_continuation(&self) -> bool {
+        if let Some(c) = self.chr0 {
+            match c {
+                '_' | '0'..='9' => true,
+                c => is_xid_continue(c),
+            }
+        } else {
+            false
+        }
+    }
+
+    /// This is the main entry point. Call this function to retrieve the next token.
+    /// This function is used by the iterator implementation.
+    fn inner_next(&mut self) -> LexResult {
+        // top loop, keep on processing, until we have something pending.
+        while self.pending.is_empty() {
+            // Detect indentation levels
+            if self.at_begin_of_line {
+                self.handle_indentations()?;
+            }
+
+            self.consume_normal()?;
+        }
+
+        Ok(self.pending.remove(0))
+    }
+
+    /// Given we are at the start of a line, count the number of spaces and/or tabs until the first character.
+    fn eat_indentation(&mut self) -> Result<IndentationLevel, LexicalError> {
+        // Determine indentation:
+        let mut spaces: usize = 0;
+        let mut tabs: usize = 0;
+        loop {
+            match self.chr0 {
+                Some(' ') => {
+                    /*
+                    if tabs != 0 {
+                        // Don't allow spaces after tabs as part of indentation.
+                        // This is technically stricter than python3 but spaces after
+                        // tabs is even more insane than mixing spaces and tabs.
+                        return Some(Err(LexicalError {
+                            error: LexicalErrorType::OtherError("Spaces not allowed as part of indentation after tabs".to_owned()),
+                            location: self.get_pos(),
+                        }));
+                    }
+                    */
+                    self.next_char();
+                    spaces += 1;
+                }
+                Some('\t') => {
+                    if spaces != 0 {
+                        // Don't allow tabs after spaces as part of indentation.
+                        // This is technically stricter than python3 but spaces before
+                        // tabs is even more insane than mixing spaces and tabs.
+                        return Err(LexicalError {
+                            error: LexicalErrorType::TabsAfterSpaces,
+                            location: self.get_pos(),
+                        });
+                    }
+                    self.next_char();
+                    tabs += 1;
+                }
+                Some('#') => {
+                    self.lex_comment();
+                    spaces = 0;
+                    tabs = 0;
+                }
+                Some('\x0C') => {
+                    // Form feed character!
+                    // Reset indentation for the Emacs user.
+                    self.next_char();
+                    spaces = 0;
+                    tabs = 0;
+                }
+                Some('\n') => {
+                    // Empty line!
+                    self.next_char();
+                    spaces = 0;
+                    tabs = 0;
+                }
+                None => {
+                    spaces = 0;
+                    tabs = 0;
+                    break;
+                }
+                _ => {
+                    self.at_begin_of_line = false;
+                    break;
+                }
+            }
+        }
+
+        Ok(IndentationLevel { tabs, spaces })
+    }
+
+    fn handle_indentations(&mut self) -> Result<(), LexicalError> {
+        let indentation_level = self.eat_indentation()?;
+
+        if self.nesting == 0 {
+            // Determine indent or dedent:
+            let current_indentation = self.indentation_stack.last().unwrap();
+            let ordering = indentation_level.compare_strict(current_indentation, self.get_pos())?;
+            match ordering {
+                Ordering::Equal => {
+                    // Same same
+                }
+                Ordering::Greater => {
+                    // New indentation level:
+                    self.indentation_stack.push(indentation_level);
+                    let tok_pos = self.get_pos();
+                    self.emit((tok_pos, Tok::Indent, tok_pos));
+                }
+                Ordering::Less => {
+                    // One or more dedentations
+                    // Pop off other levels until col is found:
+
+                    loop {
+                        let current_indentation = self.indentation_stack.last().unwrap();
+                        let ordering = indentation_level
+                            .compare_strict(current_indentation, self.get_pos())?;
+                        match ordering {
+                            Ordering::Less => {
+                                self.indentation_stack.pop();
+                                let tok_pos = self.get_pos();
+                                self.emit((tok_pos, Tok::Dedent, tok_pos));
+                            }
+                            Ordering::Equal => {
+                                // We arrived at proper level of indentation.
+                                break;
+                            }
+                            Ordering::Greater => {
+                                return Err(LexicalError {
+                                    error: LexicalErrorType::IndentationError,
+                                    location: self.get_pos(),
+                                });
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Take a look at the next character, if any, and decide upon the next steps.
+    fn consume_normal(&mut self) -> Result<(), LexicalError> {
+        // Check if we have some character:
+        if let Some(c) = self.chr0 {
+            // First check identifier:
+            if self.is_identifier_start(c) {
+                let identifier = self.lex_identifier()?;
+                self.emit(identifier);
+            } else if is_emoji_presentation(c) {
+                let tok_start = self.get_pos();
+                self.next_char();
+                let tok_end = self.get_pos();
+                self.emit((
+                    tok_start,
+                    Tok::Name {
+                        name: c.to_string(),
+                    },
+                    tok_end,
+                ));
+            } else {
+                self.consume_character(c)?;
+            }
+        } else {
+            // We reached end of file.
+            let tok_pos = self.get_pos();
+
+            // First of all, we need all nestings to be finished.
+            if self.nesting > 0 {
+                return Err(LexicalError {
+                    error: LexicalErrorType::Eof,
+                    location: tok_pos,
+                });
+            }
+
+            // Next, insert a trailing newline, if required.
+            if !self.at_begin_of_line {
+                self.at_begin_of_line = true;
+                self.emit((tok_pos, Tok::Newline, tok_pos));
+            }
+
+            // Next, flush the indentation stack to zero.
+            while self.indentation_stack.len() > 1 {
+                self.indentation_stack.pop();
+                self.emit((tok_pos, Tok::Dedent, tok_pos));
+            }
+
+            self.emit((tok_pos, Tok::EndOfFile, tok_pos));
+        }
+
+        Ok(())
+    }
+
+    /// Okay, we are facing a weird character, what is it? Determine that.
+    fn consume_character(&mut self, c: char) -> Result<(), LexicalError> {
+        match c {
+            '0'..='9' => {
+                let number = self.lex_number()?;
+                self.emit(number);
+            }
+            '#' => {
+                self.lex_comment();
+            }
+            '"' | '\'' => {
+                let string = self.lex_string(false, false, false, false)?;
+                self.emit(string);
+            }
+            '=' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                match self.chr0 {
+                    Some('=') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::EqEqual, tok_end));
+                    }
+                    _ => {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Equal, tok_end));
+                    }
+                }
+            }
+            '+' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::PlusEqual, tok_end));
+                } else {
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::Plus, tok_end));
+                }
+            }
+            '*' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                match self.chr0 {
+                    Some('=') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::StarEqual, tok_end));
+                    }
+                    Some('*') => {
+                        self.next_char();
+                        match self.chr0 {
+                            Some('=') => {
+                                self.next_char();
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::DoubleStarEqual, tok_end));
+                            }
+                            _ => {
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::DoubleStar, tok_end));
+                            }
+                        }
+                    }
+                    _ => {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Star, tok_end));
+                    }
+                }
+            }
+            '/' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                match self.chr0 {
+                    Some('=') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::SlashEqual, tok_end));
+                    }
+                    Some('/') => {
+                        self.next_char();
+                        match self.chr0 {
+                            Some('=') => {
+                                self.next_char();
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::DoubleSlashEqual, tok_end));
+                            }
+                            _ => {
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::DoubleSlash, tok_end));
+                            }
+                        }
+                    }
+                    _ => {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Slash, tok_end));
+                    }
+                }
+            }
+            '%' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::PercentEqual, tok_end));
+                } else {
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::Percent, tok_end));
+                }
+            }
+            '|' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::VbarEqual, tok_end));
+                } else {
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::Vbar, tok_end));
+                }
+            }
+            '^' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::CircumflexEqual, tok_end));
+                } else {
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::CircumFlex, tok_end));
+                }
+            }
+            '&' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::AmperEqual, tok_end));
+                } else {
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::Amper, tok_end));
+                }
+            }
+            '-' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                match self.chr0 {
+                    Some('=') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::MinusEqual, tok_end));
+                    }
+                    Some('>') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Rarrow, tok_end));
+                    }
+                    _ => {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Minus, tok_end));
+                    }
+                }
+            }
+            '@' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::AtEqual, tok_end));
+                } else {
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::At, tok_end));
+                }
+            }
+            '!' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::NotEqual, tok_end));
+                } else {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::UnrecognizedToken { tok: '!' },
+                        location: tok_start,
+                    });
+                }
+            }
+            '~' => {
+                self.eat_single_char(Tok::Tilde);
+            }
+            '(' => {
+                self.eat_single_char(Tok::Lpar);
+                self.nesting += 1;
+            }
+            ')' => {
+                self.eat_single_char(Tok::Rpar);
+                if self.nesting == 0 {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::NestingError,
+                        location: self.get_pos(),
+                    });
+                }
+                self.nesting -= 1;
+            }
+            '[' => {
+                self.eat_single_char(Tok::Lsqb);
+                self.nesting += 1;
+            }
+            ']' => {
+                self.eat_single_char(Tok::Rsqb);
+                if self.nesting == 0 {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::NestingError,
+                        location: self.get_pos(),
+                    });
+                }
+                self.nesting -= 1;
+            }
+            '{' => {
+                self.eat_single_char(Tok::Lbrace);
+                self.nesting += 1;
+            }
+            '}' => {
+                self.eat_single_char(Tok::Rbrace);
+                if self.nesting == 0 {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::NestingError,
+                        location: self.get_pos(),
+                    });
+                }
+                self.nesting -= 1;
+            }
+            ':' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                if let Some('=') = self.chr0 {
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::ColonEqual, tok_end));
+                } else {
+                    let tok_end = self.get_pos();
+                    self.emit((tok_start, Tok::Colon, tok_end));
+                }
+            }
+            ';' => {
+                self.eat_single_char(Tok::Semi);
+            }
+            '<' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                match self.chr0 {
+                    Some('<') => {
+                        self.next_char();
+                        match self.chr0 {
+                            Some('=') => {
+                                self.next_char();
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::LeftShiftEqual, tok_end));
+                            }
+                            _ => {
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::LeftShift, tok_end));
+                            }
+                        }
+                    }
+                    Some('=') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::LessEqual, tok_end));
+                    }
+                    _ => {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Less, tok_end));
+                    }
+                }
+            }
+            '>' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                match self.chr0 {
+                    Some('>') => {
+                        self.next_char();
+                        match self.chr0 {
+                            Some('=') => {
+                                self.next_char();
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::RightShiftEqual, tok_end));
+                            }
+                            _ => {
+                                let tok_end = self.get_pos();
+                                self.emit((tok_start, Tok::RightShift, tok_end));
+                            }
+                        }
+                    }
+                    Some('=') => {
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::GreaterEqual, tok_end));
+                    }
+                    _ => {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Greater, tok_end));
+                    }
+                }
+            }
+            ',' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                let tok_end = self.get_pos();
+                self.emit((tok_start, Tok::Comma, tok_end));
+            }
+            '.' => {
+                if let Some('0'..='9') = self.chr1 {
+                    let number = self.lex_number()?;
+                    self.emit(number);
+                } else {
+                    let tok_start = self.get_pos();
+                    self.next_char();
+                    if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) {
+                        self.next_char();
+                        self.next_char();
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Ellipsis, tok_end));
+                    } else {
+                        let tok_end = self.get_pos();
+                        self.emit((tok_start, Tok::Dot, tok_end));
+                    }
+                }
+            }
+            '\n' => {
+                let tok_start = self.get_pos();
+                self.next_char();
+                let tok_end = self.get_pos();
+
+                // Depending on the nesting level, we emit newline or not:
+                if self.nesting == 0 {
+                    self.at_begin_of_line = true;
+                    self.emit((tok_start, Tok::Newline, tok_end));
+                }
+            }
+            ' ' | '\t' | '\x0C' => {
+                // Skip whitespaces
+                self.next_char();
+                while self.chr0 == Some(' ') || self.chr0 == Some('\t') || self.chr0 == Some('\x0C')
+                {
+                    self.next_char();
+                }
+            }
+            '\\' => {
+                self.next_char();
+                if let Some('\n') = self.chr0 {
+                    self.next_char();
+                } else {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::LineContinuationError,
+                        location: self.get_pos(),
+                    });
+                }
+
+                if self.chr0.is_none() {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::Eof,
+                        location: self.get_pos(),
+                    });
+                }
+            }
+
+            _ => {
+                let c = self.next_char();
+                return Err(LexicalError {
+                    error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() },
+                    location: self.get_pos(),
+                });
+            } // Ignore all the rest..
+        }
+
+        Ok(())
+    }
+
+    fn eat_single_char(&mut self, ty: Tok) {
+        let tok_start = self.get_pos();
+        self.next_char().unwrap();
+        let tok_end = self.get_pos();
+        self.emit((tok_start, ty, tok_end));
+    }
+
+    /// Helper function to go to the next character coming up.
+    fn next_char(&mut self) -> Option<char> {
+        let c = self.chr0;
+        let nxt = self.chars.next();
+        self.chr0 = self.chr1;
+        self.chr1 = self.chr2;
+        self.chr2 = nxt;
+        if c == Some('\n') {
+            self.location.newline();
+        } else {
+            self.location.go_right();
+        }
+        c
+    }
+
+    /// Helper function to retrieve the current position.
+    fn get_pos(&self) -> Location {
+        self.location
+    }
+
+    /// Helper function to emit a lexed token to the queue of tokens.
+    fn emit(&mut self, spanned: Spanned) {
+        self.pending.push(spanned);
+    }
+}
+
+/* Implement iterator pattern for the get_tok function.
+
+Calling the next element in the iterator will yield the next lexical
+token.
+*/
+impl<T> Iterator for Lexer<T>
+where
+    T: Iterator<Item = char>,
+{
+    type Item = LexResult;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // Idea: create some sort of hash map for single char tokens:
+        // let mut X = HashMap::new();
+        // X.insert('=', Tok::Equal);
+        let token = self.inner_next();
+        trace!(
+            "Lex token {:?}, nesting={:?}, indent stack: {:?}",
+            token,
+            self.nesting,
+            self.indentation_stack
+        );
+
+        match token {
+            Ok((_, Tok::EndOfFile, _)) => None,
+            r => Some(r),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{make_tokenizer, NewlineHandler, StringKind, Tok};
+    use num_bigint::BigInt;
+
+    const WINDOWS_EOL: &str = "\r\n";
+    const MAC_EOL: &str = "\r";
+    const UNIX_EOL: &str = "\n";
+
+    pub fn lex_source(source: &str) -> Vec<Tok> {
+        let lexer = make_tokenizer(source);
+        lexer.map(|x| x.unwrap().1).collect()
+    }
+
+    #[test]
+    fn test_newline_processor() {
+        // Escape \ followed by \n (by removal):
+        let src = "b\\\r\n";
+        assert_eq!(4, src.len());
+        let nlh = NewlineHandler::new(src.chars());
+        let x: Vec<char> = nlh.collect();
+        assert_eq!(vec!['b', '\\', '\n'], x);
+    }
+
+    fn stok(s: &str) -> Tok {
+        Tok::String {
+            value: s.to_owned(),
+            kind: StringKind::Normal,
+        }
+    }
+
+    #[test]
+    fn test_raw_string() {
+        let source = "r\"\\\\\" \"\\\\\"";
+        let tokens = lex_source(source);
+        assert_eq!(tokens, vec![stok("\\\\"), stok("\\"), Tok::Newline,]);
+    }
+
+    #[test]
+    fn test_numbers() {
+        let source = "0x2f 0b1101 0 123 0.2 2j 2.2j";
+        let tokens = lex_source(source);
+        assert_eq!(
+            tokens,
+            vec![
+                Tok::Int {
+                    value: BigInt::from(47),
+                },
+                Tok::Int {
+                    value: BigInt::from(13),
+                },
+                Tok::Int {
+                    value: BigInt::from(0),
+                },
+                Tok::Int {
+                    value: BigInt::from(123),
+                },
+                Tok::Float { value: 0.2 },
+                Tok::Complex {
+                    real: 0.0,
+                    imag: 2.0,
+                },
+                Tok::Complex {
+                    real: 0.0,
+                    imag: 2.2,
+                },
+                Tok::Newline,
+            ]
+        );
+    }
+
+    macro_rules! test_line_comment {
+        ($($name:ident: $eol:expr,)*) => {
+            $(
+            #[test]
+            fn $name() {
+                let source = format!(r"99232  # {}", $eol);
+                let tokens = lex_source(&source);
+                assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Newline]);
+            }
+            )*
+        }
+    }
+
+    test_line_comment! {
+        test_line_comment_long: " foo",
+        test_line_comment_whitespace: "  ",
+        test_line_comment_single_whitespace: " ",
+        test_line_comment_empty: "",
+    }
+
+    macro_rules! test_comment_until_eol {
+        ($($name:ident: $eol:expr,)*) => {
+            $(
+            #[test]
+            fn $name() {
+                let source = format!("123  # Foo{}456", $eol);
+                let tokens = lex_source(&source);
+                assert_eq!(
+                    tokens,
+                    vec![
+                        Tok::Int { value: BigInt::from(123) },
+                        Tok::Newline,
+                        Tok::Int { value: BigInt::from(456) },
+                        Tok::Newline,
+                    ]
+                )
+            }
+            )*
+        }
+    }
+
+    test_comment_until_eol! {
+        test_comment_until_windows_eol: WINDOWS_EOL,
+        test_comment_until_mac_eol: MAC_EOL,
+        test_comment_until_unix_eol: UNIX_EOL,
+    }
+
+    #[test]
+    fn test_assignment() {
+        let source = r"avariable = 99 + 2-0";
+        let tokens = lex_source(source);
+        assert_eq!(
+            tokens,
+            vec![
+                Tok::Name {
+                    name: String::from("avariable"),
+                },
+                Tok::Equal,
+                Tok::Int {
+                    value: BigInt::from(99)
+                },
+                Tok::Plus,
+                Tok::Int {
+                    value: BigInt::from(2)
+                },
+                Tok::Minus,
+                Tok::Int {
+                    value: BigInt::from(0)
+                },
+                Tok::Newline,
+            ]
+        );
+    }
+
+    macro_rules! test_indentation_with_eol {
+        ($($name:ident: $eol:expr,)*) => {
+            $(
+            #[test]
+            fn $name() {
+                let source = format!("def foo():{}   return 99{}{}", $eol, $eol, $eol);
+                let tokens = lex_source(&source);
+                assert_eq!(
+                    tokens,
+                    vec![
+                        Tok::Def,
+                        Tok::Name {
+                            name: String::from("foo"),
+                        },
+                        Tok::Lpar,
+                        Tok::Rpar,
+                        Tok::Colon,
+                        Tok::Newline,
+                        Tok::Indent,
+                        Tok::Return,
+                        Tok::Int { value: BigInt::from(99) },
+                        Tok::Newline,
+                        Tok::Dedent,
+                    ]
+                );
+            }
+            )*
+        };
+    }
+
+    test_indentation_with_eol! {
+        test_indentation_windows_eol: WINDOWS_EOL,
+        test_indentation_mac_eol: MAC_EOL,
+        test_indentation_unix_eol: UNIX_EOL,
+    }
+
+    macro_rules! test_double_dedent_with_eol {
+        ($($name:ident: $eol:expr,)*) => {
+        $(
+            #[test]
+            fn $name() {
+                let source = format!("def foo():{} if x:{}{}  return 99{}{}", $eol, $eol, $eol, $eol, $eol);
+                let tokens = lex_source(&source);
+                assert_eq!(
+                    tokens,
+                    vec![
+                        Tok::Def,
+                        Tok::Name {
+                            name: String::from("foo"),
+                        },
+                        Tok::Lpar,
+                        Tok::Rpar,
+                        Tok::Colon,
+                        Tok::Newline,
+                        Tok::Indent,
+                        Tok::If,
+                        Tok::Name {
+                            name: String::from("x"),
+                        },
+                        Tok::Colon,
+                        Tok::Newline,
+                        Tok::Indent,
+                        Tok::Return,
+                        Tok::Int { value: BigInt::from(99) },
+                        Tok::Newline,
+                        Tok::Dedent,
+                        Tok::Dedent,
+                    ]
+                );
+            }
+        )*
+        }
+    }
+
+    macro_rules! test_double_dedent_with_tabs {
+        ($($name:ident: $eol:expr,)*) => {
+        $(
+            #[test]
+            fn $name() {
+                let source = format!("def foo():{}\tif x:{}{}\t return 99{}{}", $eol, $eol, $eol, $eol, $eol);
+                let tokens = lex_source(&source);
+                assert_eq!(
+                    tokens,
+                    vec![
+                        Tok::Def,
+                        Tok::Name {
+                            name: String::from("foo"),
+                        },
+                        Tok::Lpar,
+                        Tok::Rpar,
+                        Tok::Colon,
+                        Tok::Newline,
+                        Tok::Indent,
+                        Tok::If,
+                        Tok::Name {
+                            name: String::from("x"),
+                        },
+                        Tok::Colon,
+                        Tok::Newline,
+                        Tok::Indent,
+                        Tok::Return,
+                        Tok::Int { value: BigInt::from(99) },
+                        Tok::Newline,
+                        Tok::Dedent,
+                        Tok::Dedent,
+                    ]
+                );
+            }
+        )*
+        }
+    }
+
+    test_double_dedent_with_eol! {
+        test_double_dedent_windows_eol: WINDOWS_EOL,
+        test_double_dedent_mac_eol: MAC_EOL,
+        test_double_dedent_unix_eol: UNIX_EOL,
+    }
+
+    test_double_dedent_with_tabs! {
+        test_double_dedent_tabs_windows_eol: WINDOWS_EOL,
+        test_double_dedent_tabs_mac_eol: MAC_EOL,
+        test_double_dedent_tabs_unix_eol: UNIX_EOL,
+    }
+
+    macro_rules! test_newline_in_brackets {
+        ($($name:ident: $eol:expr,)*) => {
+        $(
+            #[test]
+            fn $name() {
+                let source = format!("x = [{}    1,2{}]{}", $eol, $eol, $eol);
+                let tokens = lex_source(&source);
+                assert_eq!(
+                    tokens,
+                    vec![
+                        Tok::Name {
+                            name: String::from("x"),
+                        },
+                        Tok::Equal,
+                        Tok::Lsqb,
+                        Tok::Int { value: BigInt::from(1) },
+                        Tok::Comma,
+                        Tok::Int { value: BigInt::from(2) },
+                        Tok::Rsqb,
+                        Tok::Newline,
+                    ]
+                );
+            }
+        )*
+        };
+    }
+
+    test_newline_in_brackets! {
+        test_newline_in_brackets_windows_eol: WINDOWS_EOL,
+        test_newline_in_brackets_mac_eol: MAC_EOL,
+        test_newline_in_brackets_unix_eol: UNIX_EOL,
+    }
+
+    #[test]
+    fn test_operators() {
+        let source = "//////=/ /";
+        let tokens = lex_source(source);
+        assert_eq!(
+            tokens,
+            vec![
+                Tok::DoubleSlash,
+                Tok::DoubleSlash,
+                Tok::DoubleSlashEqual,
+                Tok::Slash,
+                Tok::Slash,
+                Tok::Newline,
+            ]
+        );
+    }
+
+    #[test]
+    fn test_string() {
+        let source = r#""double" 'single' 'can\'t' "\\\"" '\t\r\n' '\g' r'raw\'' '\420' '\200\0a'"#;
+        let tokens = lex_source(source);
+        assert_eq!(
+            tokens,
+            vec![
+                stok("double"),
+                stok("single"),
+                stok("can't"),
+                stok("\\\""),
+                stok("\t\r\n"),
+                stok("\\g"),
+                stok("raw\\'"),
+                stok("Đ"),
+                stok("\u{80}\u{0}a"),
+                Tok::Newline,
+            ]
+        );
+    }
+
+    macro_rules! test_string_continuation {
+        ($($name:ident: $eol:expr,)*) => {
+        $(
+            #[test]
+            fn $name() {
+                let source = format!("\"abc\\{}def\"", $eol);
+                let tokens = lex_source(&source);
+                assert_eq!(
+                    tokens,
+                    vec![
+                        stok("abcdef"),
+                        Tok::Newline,
+                    ]
+                )
+            }
+        )*
+        }
+    }
+
+    test_string_continuation! {
+        test_string_continuation_windows_eol: WINDOWS_EOL,
+        test_string_continuation_mac_eol: MAC_EOL,
+        test_string_continuation_unix_eol: UNIX_EOL,
+    }
+
+    #[test]
+    fn test_single_quoted_byte() {
+        // single quote
+        let source = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##;
+        let tokens = lex_source(source);
+        let res = (0..=255).collect::<Vec<u8>>();
+        assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
+    }
+
+    #[test]
+    fn test_double_quoted_byte() {
+        // double quote
+        let source = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##;
+        let tokens = lex_source(source);
+        let res = (0..=255).collect::<Vec<u8>>();
+        assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
+    }
+
+    #[test]
+    fn test_escape_char_in_byte_literal() {
+        // backslash does not escape
+        let source = r##"b"omkmok\Xaa""##;
+        let tokens = lex_source(source);
+        let res = vec![111, 109, 107, 109, 111, 107, 92, 88, 97, 97];
+        assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
+    }
+
+    #[test]
+    fn test_raw_byte_literal() {
+        let source = r"rb'\x1z'";
+        let tokens = lex_source(source);
+        assert_eq!(
+            tokens,
+            vec![
+                Tok::Bytes {
+                    value: b"\\x1z".to_vec()
+                },
+                Tok::Newline
+            ]
+        );
+        let source = r"rb'\\'";
+        let tokens = lex_source(source);
+        assert_eq!(
+            tokens,
+            vec![
+                Tok::Bytes {
+                    value: b"\\\\".to_vec()
+                },
+                Tok::Newline
+            ]
+        )
+    }
+
+    #[test]
+    fn test_escape_octet() {
+        let source = r##"b'\43a\4\1234'"##;
+        let tokens = lex_source(source);
+        assert_eq!(
+            tokens,
+            vec![
+                Tok::Bytes {
+                    value: b"#a\x04S4".to_vec()
+                },
+                Tok::Newline
+            ]
+        )
+    }
+
+    #[test]
+    fn test_escape_unicode_name() {
+        let source = r#""\N{EN SPACE}""#;
+        let tokens = lex_source(source);
+        assert_eq!(tokens, vec![stok("\u{2002}"), Tok::Newline])
+    }
+}
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
new file mode 100644
index 00000000..694e0a50
--- /dev/null
+++ b/parser/src/lib.rs
@@ -0,0 +1,34 @@
+//! This crate can be used to parse python sourcecode into a so
+//! called AST (abstract syntax tree).
+//!
+//! The stages involved in this process are lexical analysis and
+//! parsing. The lexical analysis splits the sourcecode into
+//! tokens, and the parsing transforms those tokens into an AST.
+//!
+//! For example, one could do this:
+//!
+//! ```
+//! use rustpython_parser::{parser, ast};
+//!
+//! let python_source = "print('Hello world')";
+//! let python_ast = parser::parse_expression(python_source).unwrap();
+//!
+//! ```
+
+#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
+#![doc(html_root_url = "https://docs.rs/rustpython-parser/")]
+
+#[macro_use]
+extern crate log;
+pub use rustpython_ast as ast;
+
+pub mod error;
+mod fstring;
+mod function;
+pub mod lexer;
+pub mod mode;
+pub mod parser;
+#[rustfmt::skip]
+mod python;
+mod string;
+pub mod token;
diff --git a/parser/src/mode.rs b/parser/src/mode.rs
new file mode 100644
index 00000000..c03a538e
--- /dev/null
+++ b/parser/src/mode.rs
@@ -0,0 +1,40 @@
+use crate::token::Tok;
+
+#[derive(Clone, Copy)]
+pub enum Mode {
+    Module,
+    Interactive,
+    Expression,
+}
+
+impl Mode {
+    pub(crate) fn to_marker(self) -> Tok {
+        match self {
+            Self::Module => Tok::StartModule,
+            Self::Interactive => Tok::StartInteractive,
+            Self::Expression => Tok::StartExpression,
+        }
+    }
+}
+
+impl std::str::FromStr for Mode {
+    type Err = ModeParseError;
+    fn from_str(s: &str) -> Result<Self, ModeParseError> {
+        match s {
+            "exec" | "single" => Ok(Mode::Module),
+            "eval" => Ok(Mode::Expression),
+            _ => Err(ModeParseError { _priv: () }),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ModeParseError {
+    _priv: (),
+}
+
+impl std::fmt::Display for ModeParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, r#"mode should be "exec", "eval", or "single""#)
+    }
+}
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
new file mode 100644
index 00000000..990f846f
--- /dev/null
+++ b/parser/src/parser.rs
@@ -0,0 +1,202 @@
+//! Python parsing.
+//!
+//! Use this module to parse python code into an AST.
+//! There are three ways to parse python code. You could
+//! parse a whole program, a single statement, or a single
+//! expression.
+
+use std::iter;
+
+use crate::ast;
+use crate::error::ParseError;
+use crate::lexer;
+pub use crate::mode::Mode;
+use crate::python;
+
+/*
+ * Parse python code.
+ * Grammar may be inspired by antlr grammar for python:
+ * https://github.com/antlr/grammars-v4/tree/master/python3
+ */
+
+/// Parse a full python program, containing usually multiple lines.
+pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
+    parse(source, Mode::Module).map(|top| match top {
+        ast::Mod::Module { body, .. } => body,
+        _ => unreachable!(),
+    })
+}
+
+/// Parses a python expression
+///
+/// # Example
+/// ```
+/// extern crate num_bigint;
+/// use rustpython_parser::{parser, ast};
+/// let expr = parser::parse_expression("1 + 2").unwrap();
+///
+/// assert_eq!(
+///     expr,
+///     ast::Expr {
+///         location: ast::Location::new(1, 3),
+///         custom: (),
+///         node: ast::ExprKind::BinOp {
+///             left: Box::new(ast::Expr {
+///                 location: ast::Location::new(1, 1),
+///                 custom: (),
+///                 node: ast::ExprKind::Constant {
+///                     value: ast::Constant::Int(1.into()),
+///                     kind: None,
+///                 }
+///             }),
+///             op: ast::Operator::Add,
+///             right: Box::new(ast::Expr {
+///                 location: ast::Location::new(1, 5),
+///                 custom: (),
+///                 node: ast::ExprKind::Constant {
+///                     value: ast::Constant::Int(2.into()),
+///                     kind: None,
+///                 }
+///             })
+///         }
+///     },
+/// );
+///
+/// ```
+pub fn parse_expression(source: &str) -> Result<ast::Expr, ParseError> {
+    parse(source, Mode::Expression).map(|top| match top {
+        ast::Mod::Expression { body } => *body,
+        _ => unreachable!(),
+    })
+}
+
+// Parse a given source code
+pub fn parse(source: &str, mode: Mode) -> Result<ast::Mod, ParseError> {
+    let lxr = lexer::make_tokenizer(source);
+    let marker_token = (Default::default(), mode.to_marker(), Default::default());
+    let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
+
+    python::TopParser::new()
+        .parse(tokenizer)
+        .map_err(ParseError::from)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_empty() {
+        let parse_ast = parse_program("").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_string() {
+        let source = String::from("'Hello world'");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_f_string() {
+        let source = String::from("f'Hello world'");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_print_hello() {
+        let source = String::from("print('Hello world')");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_print_2() {
+        let source = String::from("print('Hello world', 2)");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_kwargs() {
+        let source = String::from("my_func('positional', keyword=2)");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_if_elif_else() {
+        let source = String::from("if 1: 10\nelif 2: 20\nelse: 30");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_lambda() {
+        let source = "lambda x, y: x * y"; // lambda(x, y): x * y";
+        let parse_ast = parse_program(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_tuples() {
+        let source = "a, b = 4, 5";
+
+        insta::assert_debug_snapshot!(parse_program(source).unwrap());
+    }
+
+    #[test]
+    fn test_parse_class() {
+        let source = "\
+class Foo(A, B):
+ def __init__(self):
+  pass
+ def method_with_default(self, arg='default'):
+  pass";
+        insta::assert_debug_snapshot!(parse_program(source).unwrap());
+    }
+
+    #[test]
+    fn test_parse_dict_comprehension() {
+        let source = String::from("{x1: x2 for y in z}");
+        let parse_ast = parse_expression(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_list_comprehension() {
+        let source = String::from("[x for y in z]");
+        let parse_ast = parse_expression(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_double_list_comprehension() {
+        let source = String::from("[x for y, y2 in z for a in b if a < 5 if a > 10]");
+        let parse_ast = parse_expression(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_generator_comprehension() {
+        let source = String::from("(x for y in z)");
+        let parse_ast = parse_expression(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_named_expression_generator_comprehension() {
+        let source = String::from("(x := y + 1 for y in z)");
+        let parse_ast = parse_expression(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_if_else_generator_comprehension() {
+        let source = String::from("(x if y else y for y in z)");
+        let parse_ast = parse_expression(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+}
diff --git a/parser/src/python.rs b/parser/src/python.rs
new file mode 100644
index 00000000..e38af9b7
--- /dev/null
+++ b/parser/src/python.rs
@@ -0,0 +1,3 @@
+#![allow(clippy::all)]
+#![allow(unused)]
+include!("../python.rs");
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
new file mode 100644
index 00000000..b2322b97
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
@@ -0,0 +1,63 @@
+---
+source: parser/src/fstring.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 0,
+        column: 0,
+    },
+    custom: (),
+    node: JoinedStr {
+        values: [
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "user=",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: FormattedValue {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 2,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "user",
+                            ctx: Load,
+                        },
+                    },
+                    conversion: 114,
+                    format_spec: None,
+                },
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
new file mode 100644
index 00000000..32d65816
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
@@ -0,0 +1,137 @@
+---
+source: parser/src/fstring.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 0,
+        column: 0,
+    },
+    custom: (),
+    node: JoinedStr {
+        values: [
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "mix ",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "user=",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: FormattedValue {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 2,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "user",
+                            ctx: Load,
+                        },
+                    },
+                    conversion: 114,
+                    format_spec: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        " with text and ",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "second=",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: FormattedValue {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 2,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "second",
+                            ctx: Load,
+                        },
+                    },
+                    conversion: 114,
+                    format_spec: None,
+                },
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
new file mode 100644
index 00000000..4ab56f07
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
@@ -0,0 +1,88 @@
+---
+source: parser/src/fstring.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 0,
+        column: 0,
+    },
+    custom: (),
+    node: JoinedStr {
+        values: [
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "user=",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: FormattedValue {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 2,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "user",
+                            ctx: Load,
+                        },
+                    },
+                    conversion: 0,
+                    format_spec: Some(
+                        Located {
+                            location: Location {
+                                row: 0,
+                                column: 0,
+                            },
+                            custom: (),
+                            node: JoinedStr {
+                                values: [
+                                    Located {
+                                        location: Location {
+                                            row: 0,
+                                            column: 0,
+                                        },
+                                        custom: (),
+                                        node: Constant {
+                                            value: Str(
+                                                ">10",
+                                            ),
+                                            kind: None,
+                                        },
+                                    },
+                                ],
+                            },
+                        },
+                    ),
+                },
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap
new file mode 100644
index 00000000..0da814cd
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap
@@ -0,0 +1,15 @@
+---
+source: parser/src/fstring.rs
+expression: "parse_fstring(\"\").unwrap()"
+
+---
+Located {
+    location: Location {
+        row: 0,
+        column: 0,
+    },
+    custom: (),
+    node: JoinedStr {
+        values: [],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
new file mode 100644
index 00000000..26af1d8e
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
@@ -0,0 +1,72 @@
+---
+source: parser/src/fstring.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 0,
+        column: 0,
+    },
+    custom: (),
+    node: JoinedStr {
+        values: [
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: FormattedValue {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 2,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "a",
+                            ctx: Load,
+                        },
+                    },
+                    conversion: 0,
+                    format_spec: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: FormattedValue {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 3,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "b",
+                            ctx: Load,
+                        },
+                    },
+                    conversion: 0,
+                    format_spec: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "{foo}",
+                    ),
+                    kind: None,
+                },
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
new file mode 100644
index 00000000..5a98ebdf
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
@@ -0,0 +1,66 @@
+---
+source: parser/src/fstring.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 0,
+        column: 0,
+    },
+    custom: (),
+    node: JoinedStr {
+        values: [
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: FormattedValue {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 5,
+                        },
+                        custom: (),
+                        node: Compare {
+                            left: Located {
+                                location: Location {
+                                    row: 1,
+                                    column: 2,
+                                },
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        42,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                            ops: [
+                                Eq,
+                            ],
+                            comparators: [
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 8,
+                                    },
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            42,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    conversion: 0,
+                    format_spec: None,
+                },
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
new file mode 100644
index 00000000..3bf74c02
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
@@ -0,0 +1,119 @@
+---
+source: parser/src/fstring.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 0,
+        column: 0,
+    },
+    custom: (),
+    node: JoinedStr {
+        values: [
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: FormattedValue {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 2,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "foo",
+                            ctx: Load,
+                        },
+                    },
+                    conversion: 0,
+                    format_spec: Some(
+                        Located {
+                            location: Location {
+                                row: 0,
+                                column: 0,
+                            },
+                            custom: (),
+                            node: JoinedStr {
+                                values: [
+                                    Located {
+                                        location: Location {
+                                            row: 0,
+                                            column: 0,
+                                        },
+                                        custom: (),
+                                        node: Constant {
+                                            value: Str(
+                                                "",
+                                            ),
+                                            kind: None,
+                                        },
+                                    },
+                                    Located {
+                                        location: Location {
+                                            row: 0,
+                                            column: 0,
+                                        },
+                                        custom: (),
+                                        node: FormattedValue {
+                                            value: Located {
+                                                location: Location {
+                                                    row: 0,
+                                                    column: 0,
+                                                },
+                                                custom: (),
+                                                node: JoinedStr {
+                                                    values: [
+                                                        Located {
+                                                            location: Location {
+                                                                row: 0,
+                                                                column: 0,
+                                                            },
+                                                            custom: (),
+                                                            node: FormattedValue {
+                                                                value: Located {
+                                                                    location: Location {
+                                                                        row: 1,
+                                                                        column: 3,
+                                                                    },
+                                                                    custom: (),
+                                                                    node: Name {
+                                                                        id: "spec",
+                                                                        ctx: Load,
+                                                                    },
+                                                                },
+                                                                conversion: 0,
+                                                                format_spec: None,
+                                                            },
+                                                        },
+                                                    ],
+                                                },
+                                            },
+                                            conversion: 0,
+                                            format_spec: None,
+                                        },
+                                    },
+                                    Located {
+                                        location: Location {
+                                            row: 0,
+                                            column: 0,
+                                        },
+                                        custom: (),
+                                        node: Constant {
+                                            value: Str(
+                                                "",
+                                            ),
+                                            kind: None,
+                                        },
+                                    },
+                                ],
+                            },
+                        },
+                    ),
+                },
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
new file mode 100644
index 00000000..655bf1b3
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
@@ -0,0 +1,66 @@
+---
+source: parser/src/fstring.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 0,
+        column: 0,
+    },
+    custom: (),
+    node: JoinedStr {
+        values: [
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: FormattedValue {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 4,
+                        },
+                        custom: (),
+                        node: Compare {
+                            left: Located {
+                                location: Location {
+                                    row: 1,
+                                    column: 2,
+                                },
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        1,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                            ops: [
+                                NotEq,
+                            ],
+                            comparators: [
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 7,
+                                    },
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            2,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    conversion: 0,
+                    format_spec: None,
+                },
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
new file mode 100644
index 00000000..8950ebbe
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
@@ -0,0 +1,62 @@
+---
+source: parser/src/fstring.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 0,
+        column: 0,
+    },
+    custom: (),
+    node: JoinedStr {
+        values: [
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: FormattedValue {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 2,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "foo",
+                            ctx: Load,
+                        },
+                    },
+                    conversion: 0,
+                    format_spec: Some(
+                        Located {
+                            location: Location {
+                                row: 0,
+                                column: 0,
+                            },
+                            custom: (),
+                            node: JoinedStr {
+                                values: [
+                                    Located {
+                                        location: Location {
+                                            row: 0,
+                                            column: 0,
+                                        },
+                                        custom: (),
+                                        node: Constant {
+                                            value: Str(
+                                                "spec",
+                                            ),
+                                            kind: None,
+                                        },
+                                    },
+                                ],
+                            },
+                        },
+                    ),
+                },
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
new file mode 100644
index 00000000..33f62eed
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
@@ -0,0 +1,63 @@
+---
+source: parser/src/fstring.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 0,
+        column: 0,
+    },
+    custom: (),
+    node: JoinedStr {
+        values: [
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "x   =",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: FormattedValue {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 2,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "x",
+                            ctx: Load,
+                        },
+                    },
+                    conversion: 114,
+                    format_spec: None,
+                },
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
new file mode 100644
index 00000000..deb7b02b
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
@@ -0,0 +1,63 @@
+---
+source: parser/src/fstring.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 0,
+        column: 0,
+    },
+    custom: (),
+    node: JoinedStr {
+        values: [
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "x=",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "   ",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: FormattedValue {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 2,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "x",
+                            ctx: Load,
+                        },
+                    },
+                    conversion: 114,
+                    format_spec: None,
+                },
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
new file mode 100644
index 00000000..37e0d05a
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
@@ -0,0 +1,36 @@
+---
+source: parser/src/fstring.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 0,
+        column: 0,
+    },
+    custom: (),
+    node: JoinedStr {
+        values: [
+            Located {
+                location: Location {
+                    row: 0,
+                    column: 0,
+                },
+                custom: (),
+                node: FormattedValue {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 2,
+                        },
+                        custom: (),
+                        node: Yield {
+                            value: None,
+                        },
+                    },
+                    conversion: 0,
+                    format_spec: None,
+                },
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
new file mode 100644
index 00000000..869669e9
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
@@ -0,0 +1,160 @@
+---
+source: parser/src/parser.rs
+expression: parse_program(&source).unwrap()
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 1,
+        },
+        custom: (),
+        node: ClassDef {
+            name: "Foo",
+            bases: [
+                Located {
+                    location: Location {
+                        row: 1,
+                        column: 11,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "A",
+                        ctx: Load,
+                    },
+                },
+                Located {
+                    location: Location {
+                        row: 1,
+                        column: 14,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "B",
+                        ctx: Load,
+                    },
+                },
+            ],
+            keywords: [],
+            body: [
+                Located {
+                    location: Location {
+                        row: 2,
+                        column: 2,
+                    },
+                    custom: (),
+                    node: FunctionDef {
+                        name: "__init__",
+                        args: Arguments {
+                            posonlyargs: [],
+                            args: [
+                                Located {
+                                    location: Location {
+                                        row: 2,
+                                        column: 15,
+                                    },
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "self",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                            ],
+                            vararg: None,
+                            kwonlyargs: [],
+                            kw_defaults: [],
+                            kwarg: None,
+                            defaults: [],
+                        },
+                        body: [
+                            Located {
+                                location: Location {
+                                    row: 3,
+                                    column: 3,
+                                },
+                                custom: (),
+                                node: Pass,
+                            },
+                        ],
+                        decorator_list: [],
+                        returns: None,
+                        type_comment: None,
+                    },
+                },
+                Located {
+                    location: Location {
+                        row: 4,
+                        column: 2,
+                    },
+                    custom: (),
+                    node: FunctionDef {
+                        name: "method_with_default",
+                        args: Arguments {
+                            posonlyargs: [],
+                            args: [
+                                Located {
+                                    location: Location {
+                                        row: 4,
+                                        column: 26,
+                                    },
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "self",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 4,
+                                        column: 32,
+                                    },
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "arg",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                            ],
+                            vararg: None,
+                            kwonlyargs: [],
+                            kw_defaults: [],
+                            kwarg: None,
+                            defaults: [
+                                Located {
+                                    location: Location {
+                                        row: 4,
+                                        column: 37,
+                                    },
+                                    custom: (),
+                                    node: Constant {
+                                        value: Str(
+                                            "default",
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                        },
+                        body: [
+                            Located {
+                                location: Location {
+                                    row: 5,
+                                    column: 3,
+                                },
+                                custom: (),
+                                node: Pass,
+                            },
+                        ],
+                        decorator_list: [],
+                        returns: None,
+                        type_comment: None,
+                    },
+                },
+            ],
+            decorator_list: [],
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap
new file mode 100644
index 00000000..f7f9b52d
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap
@@ -0,0 +1,63 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 1,
+        column: 1,
+    },
+    custom: (),
+    node: DictComp {
+        key: Located {
+            location: Location {
+                row: 1,
+                column: 2,
+            },
+            custom: (),
+            node: Name {
+                id: "x1",
+                ctx: Load,
+            },
+        },
+        value: Located {
+            location: Location {
+                row: 1,
+                column: 6,
+            },
+            custom: (),
+            node: Name {
+                id: "x2",
+                ctx: Load,
+            },
+        },
+        generators: [
+            Comprehension {
+                target: Located {
+                    location: Location {
+                        row: 1,
+                        column: 13,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "y",
+                        ctx: Load,
+                    },
+                },
+                iter: Located {
+                    location: Location {
+                        row: 1,
+                        column: 18,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "z",
+                        ctx: Load,
+                    },
+                },
+                ifs: [],
+                is_async: 0,
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
new file mode 100644
index 00000000..52d4b6ce
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
@@ -0,0 +1,178 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 1,
+        column: 1,
+    },
+    custom: (),
+    node: ListComp {
+        elt: Located {
+            location: Location {
+                row: 1,
+                column: 2,
+            },
+            custom: (),
+            node: Name {
+                id: "x",
+                ctx: Load,
+            },
+        },
+        generators: [
+            Comprehension {
+                target: Located {
+                    location: Location {
+                        row: 1,
+                        column: 8,
+                    },
+                    custom: (),
+                    node: Tuple {
+                        elts: [
+                            Located {
+                                location: Location {
+                                    row: 1,
+                                    column: 8,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "y",
+                                    ctx: Load,
+                                },
+                            },
+                            Located {
+                                location: Location {
+                                    row: 1,
+                                    column: 11,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "y2",
+                                    ctx: Load,
+                                },
+                            },
+                        ],
+                        ctx: Load,
+                    },
+                },
+                iter: Located {
+                    location: Location {
+                        row: 1,
+                        column: 17,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "z",
+                        ctx: Load,
+                    },
+                },
+                ifs: [],
+                is_async: 0,
+            },
+            Comprehension {
+                target: Located {
+                    location: Location {
+                        row: 1,
+                        column: 23,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "a",
+                        ctx: Load,
+                    },
+                },
+                iter: Located {
+                    location: Location {
+                        row: 1,
+                        column: 28,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "b",
+                        ctx: Load,
+                    },
+                },
+                ifs: [
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 35,
+                        },
+                        custom: (),
+                        node: Compare {
+                            left: Located {
+                                location: Location {
+                                    row: 1,
+                                    column: 33,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "a",
+                                    ctx: Load,
+                                },
+                            },
+                            ops: [
+                                Lt,
+                            ],
+                            comparators: [
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 37,
+                                    },
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            5,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 44,
+                        },
+                        custom: (),
+                        node: Compare {
+                            left: Located {
+                                location: Location {
+                                    row: 1,
+                                    column: 42,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "a",
+                                    ctx: Load,
+                                },
+                            },
+                            ops: [
+                                Gt,
+                            ],
+                            comparators: [
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 46,
+                                    },
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            10,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                ],
+                is_async: 0,
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap
new file mode 100644
index 00000000..64d89c15
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap
@@ -0,0 +1,5 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+---
+[]
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap
new file mode 100644
index 00000000..445e51dc
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap
@@ -0,0 +1,39 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 3,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 3,
+                },
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 3,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "Hello world",
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap
new file mode 100644
index 00000000..0e037887
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap
@@ -0,0 +1,52 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 1,
+        column: 1,
+    },
+    custom: (),
+    node: GeneratorExp {
+        elt: Located {
+            location: Location {
+                row: 1,
+                column: 2,
+            },
+            custom: (),
+            node: Name {
+                id: "x",
+                ctx: Load,
+            },
+        },
+        generators: [
+            Comprehension {
+                target: Located {
+                    location: Location {
+                        row: 1,
+                        column: 8,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "y",
+                        ctx: Load,
+                    },
+                },
+                iter: Located {
+                    location: Location {
+                        row: 1,
+                        column: 13,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "z",
+                        ctx: Load,
+                    },
+                },
+                ifs: [],
+                is_async: 0,
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
new file mode 100644
index 00000000..4b34c0e4
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
@@ -0,0 +1,125 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 1,
+        },
+        custom: (),
+        node: If {
+            test: Located {
+                location: Location {
+                    row: 1,
+                    column: 4,
+                },
+                custom: (),
+                node: Constant {
+                    value: Int(
+                        1,
+                    ),
+                    kind: None,
+                },
+            },
+            body: [
+                Located {
+                    location: Location {
+                        row: 1,
+                        column: 7,
+                    },
+                    custom: (),
+                    node: Expr {
+                        value: Located {
+                            location: Location {
+                                row: 1,
+                                column: 7,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    10,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    },
+                },
+            ],
+            orelse: [
+                Located {
+                    location: Location {
+                        row: 2,
+                        column: 1,
+                    },
+                    custom: (),
+                    node: If {
+                        test: Located {
+                            location: Location {
+                                row: 2,
+                                column: 6,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        body: [
+                            Located {
+                                location: Location {
+                                    row: 2,
+                                    column: 9,
+                                },
+                                custom: (),
+                                node: Expr {
+                                    value: Located {
+                                        location: Location {
+                                            row: 2,
+                                            column: 9,
+                                        },
+                                        custom: (),
+                                        node: Constant {
+                                            value: Int(
+                                                20,
+                                            ),
+                                            kind: None,
+                                        },
+                                    },
+                                },
+                            },
+                        ],
+                        orelse: [
+                            Located {
+                                location: Location {
+                                    row: 3,
+                                    column: 7,
+                                },
+                                custom: (),
+                                node: Expr {
+                                    value: Located {
+                                        location: Location {
+                                            row: 3,
+                                            column: 7,
+                                        },
+                                        custom: (),
+                                        node: Constant {
+                                            value: Int(
+                                                30,
+                                            ),
+                                            kind: None,
+                                        },
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            ],
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
new file mode 100644
index 00000000..782df39a
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
@@ -0,0 +1,83 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 1,
+        column: 1,
+    },
+    custom: (),
+    node: GeneratorExp {
+        elt: Located {
+            location: Location {
+                row: 1,
+                column: 4,
+            },
+            custom: (),
+            node: IfExp {
+                test: Located {
+                    location: Location {
+                        row: 1,
+                        column: 7,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "y",
+                        ctx: Load,
+                    },
+                },
+                body: Located {
+                    location: Location {
+                        row: 1,
+                        column: 2,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "x",
+                        ctx: Load,
+                    },
+                },
+                orelse: Located {
+                    location: Location {
+                        row: 1,
+                        column: 14,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "y",
+                        ctx: Load,
+                    },
+                },
+            },
+        },
+        generators: [
+            Comprehension {
+                target: Located {
+                    location: Location {
+                        row: 1,
+                        column: 20,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "y",
+                        ctx: Load,
+                    },
+                },
+                iter: Located {
+                    location: Location {
+                        row: 1,
+                        column: 25,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "z",
+                        ctx: Load,
+                    },
+                },
+                ifs: [],
+                is_async: 0,
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap
new file mode 100644
index 00000000..d31a3921
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap
@@ -0,0 +1,78 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 1,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 8,
+                },
+                custom: (),
+                node: Call {
+                    func: Located {
+                        location: Location {
+                            row: 1,
+                            column: 1,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "my_func",
+                            ctx: Load,
+                        },
+                    },
+                    args: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 10,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "positional",
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    keywords: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 23,
+                            },
+                            custom: (),
+                            node: KeywordData {
+                                arg: Some(
+                                    "keyword",
+                                ),
+                                value: Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 31,
+                                    },
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            2,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap
new file mode 100644
index 00000000..2f70292d
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap
@@ -0,0 +1,90 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 1,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 1,
+                },
+                custom: (),
+                node: Lambda {
+                    args: Arguments {
+                        posonlyargs: [],
+                        args: [
+                            Located {
+                                location: Location {
+                                    row: 1,
+                                    column: 8,
+                                },
+                                custom: (),
+                                node: ArgData {
+                                    arg: "x",
+                                    annotation: None,
+                                    type_comment: None,
+                                },
+                            },
+                            Located {
+                                location: Location {
+                                    row: 1,
+                                    column: 11,
+                                },
+                                custom: (),
+                                node: ArgData {
+                                    arg: "y",
+                                    annotation: None,
+                                    type_comment: None,
+                                },
+                            },
+                        ],
+                        vararg: None,
+                        kwonlyargs: [],
+                        kw_defaults: [],
+                        kwarg: None,
+                        defaults: [],
+                    },
+                    body: Located {
+                        location: Location {
+                            row: 1,
+                            column: 16,
+                        },
+                        custom: (),
+                        node: BinOp {
+                            left: Located {
+                                location: Location {
+                                    row: 1,
+                                    column: 14,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "x",
+                                    ctx: Load,
+                                },
+                            },
+                            op: Mult,
+                            right: Located {
+                                location: Location {
+                                    row: 1,
+                                    column: 18,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "y",
+                                    ctx: Load,
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap
new file mode 100644
index 00000000..8a6d8eb8
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap
@@ -0,0 +1,52 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 1,
+        column: 1,
+    },
+    custom: (),
+    node: ListComp {
+        elt: Located {
+            location: Location {
+                row: 1,
+                column: 2,
+            },
+            custom: (),
+            node: Name {
+                id: "x",
+                ctx: Load,
+            },
+        },
+        generators: [
+            Comprehension {
+                target: Located {
+                    location: Location {
+                        row: 1,
+                        column: 8,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "y",
+                        ctx: Load,
+                    },
+                },
+                iter: Located {
+                    location: Location {
+                        row: 1,
+                        column: 13,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "z",
+                        ctx: Load,
+                    },
+                },
+                ifs: [],
+                is_async: 0,
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
new file mode 100644
index 00000000..960151b0
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
@@ -0,0 +1,95 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 1,
+        column: 1,
+    },
+    custom: (),
+    node: GeneratorExp {
+        elt: Located {
+            location: Location {
+                row: 1,
+                column: 2,
+            },
+            custom: (),
+            node: NamedExpr {
+                target: Located {
+                    location: Location {
+                        row: 1,
+                        column: 2,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "x",
+                        ctx: Store,
+                    },
+                },
+                value: Located {
+                    location: Location {
+                        row: 1,
+                        column: 9,
+                    },
+                    custom: (),
+                    node: BinOp {
+                        left: Located {
+                            location: Location {
+                                row: 1,
+                                column: 7,
+                            },
+                            custom: (),
+                            node: Name {
+                                id: "y",
+                                ctx: Load,
+                            },
+                        },
+                        op: Add,
+                        right: Located {
+                            location: Location {
+                                row: 1,
+                                column: 11,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    },
+                },
+            },
+        },
+        generators: [
+            Comprehension {
+                target: Located {
+                    location: Location {
+                        row: 1,
+                        column: 17,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "y",
+                        ctx: Load,
+                    },
+                },
+                iter: Located {
+                    location: Location {
+                        row: 1,
+                        column: 22,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "z",
+                        ctx: Load,
+                    },
+                },
+                ifs: [],
+                is_async: 0,
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap
new file mode 100644
index 00000000..6fe4cad3
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap
@@ -0,0 +1,65 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 1,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 6,
+                },
+                custom: (),
+                node: Call {
+                    func: Located {
+                        location: Location {
+                            row: 1,
+                            column: 1,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "print",
+                            ctx: Load,
+                        },
+                    },
+                    args: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 8,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "Hello world",
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 22,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    keywords: [],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap
new file mode 100644
index 00000000..b4e78d8d
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap
@@ -0,0 +1,51 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 1,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 6,
+                },
+                custom: (),
+                node: Call {
+                    func: Located {
+                        location: Location {
+                            row: 1,
+                            column: 1,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "print",
+                            ctx: Load,
+                        },
+                    },
+                    args: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 8,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "Hello world",
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    keywords: [],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap
new file mode 100644
index 00000000..b808f28e
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap
@@ -0,0 +1,28 @@
+---
+source: parser/src/parser.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 2,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 2,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "Hello world",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap
new file mode 100644
index 00000000..26243f56
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap
@@ -0,0 +1,91 @@
+---
+source: parser/src/parser.rs
+expression: parse_program(&source).unwrap()
+
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 1,
+        },
+        custom: (),
+        node: Assign {
+            targets: [
+                Located {
+                    location: Location {
+                        row: 1,
+                        column: 1,
+                    },
+                    custom: (),
+                    node: Tuple {
+                        elts: [
+                            Located {
+                                location: Location {
+                                    row: 1,
+                                    column: 1,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "a",
+                                    ctx: Load,
+                                },
+                            },
+                            Located {
+                                location: Location {
+                                    row: 1,
+                                    column: 4,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "b",
+                                    ctx: Load,
+                                },
+                            },
+                        ],
+                        ctx: Load,
+                    },
+                },
+            ],
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 8,
+                },
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 8,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    4,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 11,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    5,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+            type_comment: None,
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
new file mode 100644
index 00000000..687ee960
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
@@ -0,0 +1,39 @@
+---
+source: parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 2,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 2,
+                },
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 2,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "Hello world",
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
new file mode 100644
index 00000000..687ee960
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
@@ -0,0 +1,39 @@
+---
+source: parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 2,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 2,
+                },
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 2,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "Hello world",
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
new file mode 100644
index 00000000..197f1c57
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
@@ -0,0 +1,63 @@
+---
+source: parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 2,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 2,
+                },
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 2,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "Hello world",
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            custom: (),
+                            node: FormattedValue {
+                                value: Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 3,
+                                    },
+                                    custom: (),
+                                    node: Constant {
+                                        value: Str(
+                                            "!",
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                conversion: 0,
+                                format_spec: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
new file mode 100644
index 00000000..25ed2337
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
@@ -0,0 +1,28 @@
+---
+source: parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 2,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 2,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "Hello world",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
new file mode 100644
index 00000000..bcba0524
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
@@ -0,0 +1,41 @@
+---
+source: parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 3,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 3,
+                },
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 3,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "Hello world",
+                                ),
+                                kind: Some(
+                                    "u",
+                                ),
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
new file mode 100644
index 00000000..4e4ba23d
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
@@ -0,0 +1,41 @@
+---
+source: parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 3,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 3,
+                },
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 3,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "Hello world!",
+                                ),
+                                kind: Some(
+                                    "u",
+                                ),
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
new file mode 100644
index 00000000..25ed2337
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
@@ -0,0 +1,28 @@
+---
+source: parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 2,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 2,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "Hello world",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
new file mode 100644
index 00000000..190854c3
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
@@ -0,0 +1,30 @@
+---
+source: parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 3,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 3,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "Hello world",
+                    ),
+                    kind: Some(
+                        "u",
+                    ),
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/string.rs b/parser/src/string.rs
new file mode 100644
index 00000000..02a55145
--- /dev/null
+++ b/parser/src/string.rs
@@ -0,0 +1,143 @@
+use crate::{
+    ast::{Constant, Expr, ExprKind, Location},
+    error::{LexicalError, LexicalErrorType},
+    fstring::parse_located_fstring,
+    token::StringKind,
+};
+use itertools::Itertools;
+
+pub fn parse_strings(values: Vec<(Location, (String, StringKind))>) -> Result<Expr, LexicalError> {
+    // Preserve the initial location and kind.
+    let initial_location = values[0].0;
+    let initial_kind = (values[0].1 .1 == StringKind::U).then(|| "u".to_owned());
+
+    // Determine whether the list of values contains any f-strings. (If not, we can return a
+    // single Constant at the end, rather than a JoinedStr.)
+    let mut has_fstring = false;
+
+    // De-duplicate adjacent constants.
+    let mut deduped: Vec<Expr> = vec![];
+    let mut current: Vec<String> = vec![];
+
+    let take_current = |current: &mut Vec<String>| -> Expr {
+        Expr::new(
+            initial_location,
+            ExprKind::Constant {
+                value: Constant::Str(current.drain(..).join("")),
+                kind: initial_kind.clone(),
+            },
+        )
+    };
+
+    for (location, (string, string_kind)) in values {
+        match string_kind {
+            StringKind::Normal | StringKind::U => current.push(string),
+            StringKind::F => {
+                has_fstring = true;
+                let values = if let ExprKind::JoinedStr { values } =
+                    parse_located_fstring(&string, location)
+                        .map_err(|e| LexicalError {
+                            location,
+                            error: LexicalErrorType::FStringError(e.error),
+                        })?
+                        .node
+                {
+                    values
+                } else {
+                    unreachable!("parse_located_fstring returned a non-JoinedStr.")
+                };
+                for value in values {
+                    match value.node {
+                        ExprKind::FormattedValue { .. } => {
+                            if !current.is_empty() {
+                                deduped.push(take_current(&mut current));
+                            }
+                            deduped.push(value)
+                        }
+                        ExprKind::Constant { value, .. } => {
+                            if let Constant::Str(value) = value {
+                                current.push(value);
+                            } else {
+                                unreachable!("Unexpected non-string constant.");
+                            }
+                        }
+                        _ => unreachable!("Unexpected non-string expression."),
+                    }
+                }
+            }
+        }
+    }
+    if !current.is_empty() {
+        deduped.push(take_current(&mut current));
+    }
+
+    Ok(if has_fstring {
+        Expr::new(initial_location, ExprKind::JoinedStr { values: deduped })
+    } else {
+        deduped
+            .into_iter()
+            .exactly_one()
+            .expect("String must be concatenated to a single element.")
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::parser::parse_program;
+
+    #[test]
+    fn test_parse_string_concat() {
+        let source = String::from("'Hello ' 'world'");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_u_string_concat_1() {
+        let source = String::from("'Hello ' u'world'");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_u_string_concat_2() {
+        let source = String::from("u'Hello ' 'world'");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_f_string_concat_1() {
+        let source = String::from("'Hello ' f'world'");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_f_string_concat_2() {
+        let source = String::from("'Hello ' f'world'");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_f_string_concat_3() {
+        let source = String::from("'Hello ' f'world{\"!\"}'");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_u_f_string_concat_1() {
+        let source = String::from("u'Hello ' f'world'");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_u_f_string_concat_2() {
+        let source = String::from("u'Hello ' f'world' '!'");
+        let parse_ast = parse_program(&source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+}
diff --git a/parser/src/token.rs b/parser/src/token.rs
new file mode 100644
index 00000000..5fd6728b
--- /dev/null
+++ b/parser/src/token.rs
@@ -0,0 +1,236 @@
+//! Different token definitions.
+//! Loosely based on token.h from CPython source:
+use num_bigint::BigInt;
+use std::fmt::{self, Write};
+
+/// Python source code can be tokenized in a sequence of these tokens.
+#[derive(Clone, Debug, PartialEq)]
+pub enum Tok {
+    Name { name: String },
+    Int { value: BigInt },
+    Float { value: f64 },
+    Complex { real: f64, imag: f64 },
+    String { value: String, kind: StringKind },
+    Bytes { value: Vec<u8> },
+    Newline,
+    Indent,
+    Dedent,
+    StartModule,
+    StartInteractive,
+    StartExpression,
+    EndOfFile,
+    Lpar,
+    Rpar,
+    Lsqb,
+    Rsqb,
+    Colon,
+    Comma,
+    Semi,
+    Plus,
+    Minus,
+    Star,
+    Slash,
+    Vbar,  // '|'
+    Amper, // '&'
+    Less,
+    Greater,
+    Equal,
+    Dot,
+    Percent,
+    Lbrace,
+    Rbrace,
+    EqEqual,
+    NotEqual,
+    LessEqual,
+    GreaterEqual,
+    Tilde,
+    CircumFlex,
+    LeftShift,
+    RightShift,
+    DoubleStar,
+    DoubleStarEqual, // '**='
+    PlusEqual,
+    MinusEqual,
+    StarEqual,
+    SlashEqual,
+    PercentEqual,
+    AmperEqual, // '&='
+    VbarEqual,
+    CircumflexEqual, // '^='
+    LeftShiftEqual,
+    RightShiftEqual,
+    DoubleSlash, // '//'
+    DoubleSlashEqual,
+    ColonEqual,
+    At,
+    AtEqual,
+    Rarrow,
+    Ellipsis,
+
+    // Keywords (alphabetically):
+    False,
+    None,
+    True,
+
+    And,
+    As,
+    Assert,
+    Async,
+    Await,
+    Break,
+    Class,
+    Continue,
+    Def,
+    Del,
+    Elif,
+    Else,
+    Except,
+    Finally,
+    For,
+    From,
+    Global,
+    If,
+    Import,
+    In,
+    Is,
+    Lambda,
+    Nonlocal,
+    Not,
+    Or,
+    Pass,
+    Raise,
+    Return,
+    Try,
+    While,
+    With,
+    Yield,
+}
+
+#[derive(PartialEq, Eq, Debug, Clone)]
+pub enum StringKind {
+    Normal,
+    F,
+    U,
+}
+
+impl fmt::Display for Tok {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use Tok::*;
+        match self {
+            Name { name } => write!(f, "'{}'", name),
+            Int { value } => write!(f, "'{}'", value),
+            Float { value } => write!(f, "'{}'", value),
+            Complex { real, imag } => write!(f, "{}j{}", real, imag),
+            String { value, kind } => {
+                match kind {
+                    StringKind::F => f.write_str("f")?,
+                    StringKind::U => f.write_str("u")?,
+                    StringKind::Normal => {}
+                }
+                write!(f, "{:?}", value)
+            }
+            Bytes { value } => {
+                write!(f, "b\"")?;
+                for i in value {
+                    match i {
+                        9 => f.write_str("\\t")?,
+                        10 => f.write_str("\\n")?,
+                        13 => f.write_str("\\r")?,
+                        32..=126 => f.write_char(*i as char)?,
+                        _ => write!(f, "\\x{:02x}", i)?,
+                    }
+                }
+                f.write_str("\"")
+            }
+            Newline => f.write_str("Newline"),
+            Indent => f.write_str("Indent"),
+            Dedent => f.write_str("Dedent"),
+            StartModule => f.write_str("StartProgram"),
+            StartInteractive => f.write_str("StartInteractive"),
+            StartExpression => f.write_str("StartExpression"),
+            EndOfFile => f.write_str("EOF"),
+            Lpar => f.write_str("'('"),
+            Rpar => f.write_str("')'"),
+            Lsqb => f.write_str("'['"),
+            Rsqb => f.write_str("']'"),
+            Colon => f.write_str("':'"),
+            Comma => f.write_str("','"),
+            Semi => f.write_str("';'"),
+            Plus => f.write_str("'+'"),
+            Minus => f.write_str("'-'"),
+            Star => f.write_str("'*'"),
+            Slash => f.write_str("'/'"),
+            Vbar => f.write_str("'|'"),
+            Amper => f.write_str("'&'"),
+            Less => f.write_str("'<'"),
+            Greater => f.write_str("'>'"),
+            Equal => f.write_str("'='"),
+            Dot => f.write_str("'.'"),
+            Percent => f.write_str("'%'"),
+            Lbrace => f.write_str("'{'"),
+            Rbrace => f.write_str("'}'"),
+            EqEqual => f.write_str("'=='"),
+            NotEqual => f.write_str("'!='"),
+            LessEqual => f.write_str("'<='"),
+            GreaterEqual => f.write_str("'>='"),
+            Tilde => f.write_str("'~'"),
+            CircumFlex => f.write_str("'^'"),
+            LeftShift => f.write_str("'<<'"),
+            RightShift => f.write_str("'>>'"),
+            DoubleStar => f.write_str("'**'"),
+            DoubleStarEqual => f.write_str("'**='"),
+            PlusEqual => f.write_str("'+='"),
+            MinusEqual => f.write_str("'-='"),
+            StarEqual => f.write_str("'*='"),
+            SlashEqual => f.write_str("'/='"),
+            PercentEqual => f.write_str("'%='"),
+            AmperEqual => f.write_str("'&='"),
+            VbarEqual => f.write_str("'|='"),
+            CircumflexEqual => f.write_str("'^='"),
+            LeftShiftEqual => f.write_str("'<<='"),
+            RightShiftEqual => f.write_str("'>>='"),
+            DoubleSlash => f.write_str("'//'"),
+            DoubleSlashEqual => f.write_str("'//='"),
+            At => f.write_str("'@'"),
+            AtEqual => f.write_str("'@='"),
+            Rarrow => f.write_str("'->'"),
+            Ellipsis => f.write_str("'...'"),
+            False => f.write_str("'False'"),
+            None => f.write_str("'None'"),
+            True => f.write_str("'True'"),
+            And => f.write_str("'and'"),
+            As => f.write_str("'as'"),
+            Assert => f.write_str("'assert'"),
+            Async => f.write_str("'async'"),
+            Await => f.write_str("'await'"),
+            Break => f.write_str("'break'"),
+            Class => f.write_str("'class'"),
+            Continue => f.write_str("'continue'"),
+            Def => f.write_str("'def'"),
+            Del => f.write_str("'del'"),
+            Elif => f.write_str("'elif'"),
+            Else => f.write_str("'else'"),
+            Except => f.write_str("'except'"),
+            Finally => f.write_str("'finally'"),
+            For => f.write_str("'for'"),
+            From => f.write_str("'from'"),
+            Global => f.write_str("'global'"),
+            If => f.write_str("'if'"),
+            Import => f.write_str("'import'"),
+            In => f.write_str("'in'"),
+            Is => f.write_str("'is'"),
+            Lambda => f.write_str("'lambda'"),
+            Nonlocal => f.write_str("'nonlocal'"),
+            Not => f.write_str("'not'"),
+            Or => f.write_str("'or'"),
+            Pass => f.write_str("'pass'"),
+            Raise => f.write_str("'raise'"),
+            Return => f.write_str("'return'"),
+            Try => f.write_str("'try'"),
+            While => f.write_str("'while'"),
+            With => f.write_str("'with'"),
+            Yield => f.write_str("'yield'"),
+            ColonEqual => f.write_str("':='"),
+        }
+    }
+}
diff --git a/porcelain/Cargo.toml b/porcelain/Cargo.toml
deleted file mode 100644
index 56bf737e..00000000
--- a/porcelain/Cargo.toml
+++ /dev/null
@@ -1,12 +0,0 @@
-[package]
-name = "rustpython-compiler"
-version = "0.1.2"
-description = "A usability wrapper around rustpython-parser and rustpython-compiler-core"
-authors = ["RustPython Team"]
-edition = "2021"
-
-[dependencies]
-thiserror = "1.0"
-rustpython-compiler-core = { path = ".." }
-rustpython-parser = { path = "../../parser" }
-rustpython-bytecode = { path = "../../bytecode" }
diff --git a/porcelain/src/lib.rs b/porcelain/src/lib.rs
deleted file mode 100644
index fcbcfa87..00000000
--- a/porcelain/src/lib.rs
+++ /dev/null
@@ -1,124 +0,0 @@
-use rustpython_bytecode::CodeObject;
-use rustpython_compiler_core::{compile, symboltable};
-use rustpython_parser::{
-    ast::{fold::Fold, ConstantOptimizer, Location},
-    parser,
-};
-use std::fmt;
-
-pub use compile::{CompileOpts, Mode};
-pub use symboltable::{Symbol, SymbolScope, SymbolTable, SymbolTableType};
-
-#[derive(Debug, thiserror::Error)]
-pub enum CompileErrorType {
-    #[error(transparent)]
-    Compile(#[from] rustpython_compiler_core::error::CompileErrorType),
-    #[error(transparent)]
-    Parse(#[from] rustpython_parser::error::ParseErrorType),
-}
-
-#[derive(Debug, thiserror::Error)]
-pub struct CompileError {
-    pub error: CompileErrorType,
-    pub statement: Option<String>,
-    pub source_path: String,
-    pub location: Location,
-}
-
-impl fmt::Display for CompileError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let loc = self.location;
-        if let Some(ref stmt) = self.statement {
-            // visualize the error when location and statement are provided
-            write!(
-                f,
-                "{}",
-                loc.visualize(stmt, &format_args!("{} at {}", self.error, loc))
-            )
-        } else {
-            write!(f, "{} at {}", self.error, loc)
-        }
-    }
-}
-
-impl CompileError {
-    fn from_compile(error: rustpython_compiler_core::error::CompileError, source: &str) -> Self {
-        CompileError {
-            error: error.error.into(),
-            location: error.location,
-            source_path: error.source_path,
-            statement: get_statement(source, error.location),
-        }
-    }
-    fn from_parse(
-        error: rustpython_parser::error::ParseError,
-        source: &str,
-        source_path: String,
-    ) -> Self {
-        CompileError {
-            error: error.error.into(),
-            location: error.location,
-            source_path,
-            statement: get_statement(source, error.location),
-        }
-    }
-    fn from_symtable(
-        error: symboltable::SymbolTableError,
-        source: &str,
-        source_path: String,
-    ) -> Self {
-        Self::from_compile(error.into_compile_error(source_path), source)
-    }
-}
-
-/// Compile a given sourcecode into a bytecode object.
-pub fn compile(
-    source: &str,
-    mode: compile::Mode,
-    source_path: String,
-    opts: CompileOpts,
-) -> Result<CodeObject, CompileError> {
-    let parser_mode = match mode {
-        compile::Mode::Exec => parser::Mode::Module,
-        compile::Mode::Eval => parser::Mode::Expression,
-        compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive,
-    };
-    let mut ast = match parser::parse(source, parser_mode) {
-        Ok(x) => x,
-        Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
-    };
-    if opts.optimize > 0 {
-        ast = ConstantOptimizer::new()
-            .fold_mod(ast)
-            .unwrap_or_else(|e| match e {});
-    }
-    compile::compile_top(&ast, source_path, mode, opts)
-        .map_err(|e| CompileError::from_compile(e, source))
-}
-
-pub fn compile_symtable(
-    source: &str,
-    mode: compile::Mode,
-    source_path: &str,
-) -> Result<symboltable::SymbolTable, CompileError> {
-    let parse_err = |e| CompileError::from_parse(e, source, source_path.to_owned());
-    let res = match mode {
-        compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
-            let ast = parser::parse_program(source).map_err(parse_err)?;
-            symboltable::make_symbol_table(&ast)
-        }
-        compile::Mode::Eval => {
-            let expr = parser::parse_expression(source).map_err(parse_err)?;
-            symboltable::make_symbol_table_expr(&expr)
-        }
-    };
-    res.map_err(|e| CompileError::from_symtable(e, source, source_path.to_owned()))
-}
-
-fn get_statement(source: &str, loc: Location) -> Option<String> {
-    if loc.column() == 0 || loc.row() == 0 {
-        return None;
-    }
-    let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
-    Some(line + "\n")
-}
diff --git a/src/lib.rs b/src/lib.rs
index 3e3aca8f..1a659a88 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,15 +1,124 @@
-//! Compile a Python AST or source code into bytecode consumable by RustPython.
-#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
-#![doc(html_root_url = "https://docs.rs/rustpython-compiler/")]
+use rustpython_bytecode::CodeObject;
+use rustpython_codegen::{compile, symboltable};
+use rustpython_parser::{
+    ast::{fold::Fold, ConstantOptimizer, Location},
+    parser,
+};
+use std::fmt;
 
-#[macro_use]
-extern crate log;
+pub use compile::{CompileOpts, Mode};
+pub use symboltable::{Symbol, SymbolScope, SymbolTable, SymbolTableType};
 
-type IndexMap<K, V> = indexmap::IndexMap<K, V, ahash::RandomState>;
-type IndexSet<T> = indexmap::IndexSet<T, ahash::RandomState>;
+#[derive(Debug, thiserror::Error)]
+pub enum CompileErrorType {
+    #[error(transparent)]
+    Compile(#[from] rustpython_codegen::error::CompileErrorType),
+    #[error(transparent)]
+    Parse(#[from] rustpython_parser::error::ParseErrorType),
+}
 
-pub mod compile;
-pub mod error;
-pub mod ir;
-pub mod mode;
-pub mod symboltable;
+#[derive(Debug, thiserror::Error)]
+pub struct CompileError {
+    pub error: CompileErrorType,
+    pub statement: Option<String>,
+    pub source_path: String,
+    pub location: Location,
+}
+
+impl fmt::Display for CompileError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let loc = self.location;
+        if let Some(ref stmt) = self.statement {
+            // visualize the error when location and statement are provided
+            write!(
+                f,
+                "{}",
+                loc.visualize(stmt, &format_args!("{} at {}", self.error, loc))
+            )
+        } else {
+            write!(f, "{} at {}", self.error, loc)
+        }
+    }
+}
+
+impl CompileError {
+    fn from_compile(error: rustpython_codegen::error::CompileError, source: &str) -> Self {
+        CompileError {
+            error: error.error.into(),
+            location: error.location,
+            source_path: error.source_path,
+            statement: get_statement(source, error.location),
+        }
+    }
+    fn from_parse(
+        error: rustpython_parser::error::ParseError,
+        source: &str,
+        source_path: String,
+    ) -> Self {
+        CompileError {
+            error: error.error.into(),
+            location: error.location,
+            source_path,
+            statement: get_statement(source, error.location),
+        }
+    }
+    fn from_symtable(
+        error: symboltable::SymbolTableError,
+        source: &str,
+        source_path: String,
+    ) -> Self {
+        Self::from_compile(error.into_compile_error(source_path), source)
+    }
+}
+
+/// Compile a given sourcecode into a bytecode object.
+pub fn compile(
+    source: &str,
+    mode: compile::Mode,
+    source_path: String,
+    opts: CompileOpts,
+) -> Result<CodeObject, CompileError> {
+    let parser_mode = match mode {
+        compile::Mode::Exec => parser::Mode::Module,
+        compile::Mode::Eval => parser::Mode::Expression,
+        compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive,
+    };
+    let mut ast = match parser::parse(source, parser_mode) {
+        Ok(x) => x,
+        Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
+    };
+    if opts.optimize > 0 {
+        ast = ConstantOptimizer::new()
+            .fold_mod(ast)
+            .unwrap_or_else(|e| match e {});
+    }
+    compile::compile_top(&ast, source_path, mode, opts)
+        .map_err(|e| CompileError::from_compile(e, source))
+}
+
+pub fn compile_symtable(
+    source: &str,
+    mode: compile::Mode,
+    source_path: &str,
+) -> Result<symboltable::SymbolTable, CompileError> {
+    let parse_err = |e| CompileError::from_parse(e, source, source_path.to_owned());
+    let res = match mode {
+        compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
+            let ast = parser::parse_program(source).map_err(parse_err)?;
+            symboltable::make_symbol_table(&ast)
+        }
+        compile::Mode::Eval => {
+            let expr = parser::parse_expression(source).map_err(parse_err)?;
+            symboltable::make_symbol_table_expr(&expr)
+        }
+    };
+    res.map_err(|e| CompileError::from_symtable(e, source, source_path.to_owned()))
+}
+
+fn get_statement(source: &str, loc: Location) -> Option<String> {
+    if loc.column() == 0 || loc.row() == 0 {
+        return None;
+    }
+    let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
+    Some(line + "\n")
+}

From fa2e69983f4ca885d16bd45bc3231d0df8716b44 Mon Sep 17 00:00:00 2001
From: Charles Marsh <charlie@springdisc.com>
Date: Fri, 19 Aug 2022 10:47:37 -0400
Subject: [PATCH 251/459] Avoid creating unused JoinedStr in FStringParser

---
 parser/src/fstring.rs                         |  52 ++--
 ...s__fstring_parse_selfdocumenting_base.snap |  95 ++++---
 ...tring_parse_selfdocumenting_base_more.snap | 233 +++++++++---------
 ..._fstring_parse_selfdocumenting_format.snap | 139 +++++------
 ...__fstring__tests__parse_empty_fstring.snap |  12 +-
 ...parser__fstring__tests__parse_fstring.snap | 105 ++++----
 ..._fstring__tests__parse_fstring_equals.snap |  91 +++----
 ...ing__tests__parse_fstring_nested_spec.snap | 150 ++++-------
 ...ring__tests__parse_fstring_not_equals.snap |  91 +++----
 ..._tests__parse_fstring_not_nested_spec.snap |  93 ++++---
 ...sts__parse_fstring_selfdoc_prec_space.snap |  95 ++++---
 ..._parse_fstring_selfdoc_trailing_space.snap |  95 ++++---
 ...ring__tests__parse_fstring_yield_expr.snap |  43 ++--
 parser/src/string.rs                          |  17 +-
 14 files changed, 567 insertions(+), 744 deletions(-)

diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index 28693061..0d5e2605 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -105,19 +105,13 @@ impl<'a> FStringParser<'a> {
                                 nested -= 1;
                                 if nested == 0 {
                                     formatted_value_piece.push(next);
-                                    spec_constructor.push(
-                                        self.expr(ExprKind::FormattedValue {
-                                            value: Box::new(
-                                                FStringParser::new(
-                                                    &formatted_value_piece,
-                                                    Location::default(),
-                                                    &self.recurse_lvl + 1,
-                                                )
-                                                .parse()?,
-                                            ),
-                                            conversion: ConversionFlag::None as _,
-                                            format_spec: None,
-                                        }),
+                                    spec_constructor.extend(
+                                        FStringParser::new(
+                                            &format!("{{{}}}", formatted_value_piece),
+                                            Location::default(),
+                                            &self.recurse_lvl + 1,
+                                        )
+                                        .parse()?,
                                     );
                                     formatted_value_piece.clear();
                                 } else {
@@ -129,11 +123,13 @@ impl<'a> FStringParser<'a> {
                             }
                             '{' => {
                                 nested += 1;
-                                spec_constructor.push(self.expr(ExprKind::Constant {
-                                    value: constant_piece.to_owned().into(),
-                                    kind: None,
-                                }));
-                                constant_piece.clear();
+                                if !constant_piece.is_empty() {
+                                    spec_constructor.push(self.expr(ExprKind::Constant {
+                                        value: constant_piece.to_owned().into(),
+                                        kind: None,
+                                    }));
+                                    constant_piece.clear();
+                                }
                                 formatted_value_piece.push(next);
                                 formatted_value_piece.push(' ');
                             }
@@ -144,11 +140,13 @@ impl<'a> FStringParser<'a> {
                         }
                         self.chars.next();
                     }
-                    spec_constructor.push(self.expr(ExprKind::Constant {
-                        value: constant_piece.to_owned().into(),
-                        kind: None,
-                    }));
-                    constant_piece.clear();
+                    if !constant_piece.is_empty() {
+                        spec_constructor.push(self.expr(ExprKind::Constant {
+                            value: constant_piece.to_owned().into(),
+                            kind: None,
+                        }));
+                        constant_piece.clear();
+                    }
                     if nested > 0 {
                         return Err(UnclosedLbrace);
                     }
@@ -241,7 +239,7 @@ impl<'a> FStringParser<'a> {
         Err(UnclosedLbrace)
     }
 
-    fn parse(mut self) -> Result<Expr, FStringErrorType> {
+    fn parse(mut self) -> Result<Vec<Expr>, FStringErrorType> {
         if self.recurse_lvl >= 2 {
             return Err(ExpressionNestedTooDeeply);
         }
@@ -287,7 +285,7 @@ impl<'a> FStringParser<'a> {
             }))
         }
 
-        Ok(self.expr(ExprKind::JoinedStr { values }))
+        Ok(values)
     }
 }
 
@@ -298,7 +296,7 @@ fn parse_fstring_expr(source: &str) -> Result<Expr, ParseError> {
 
 /// Parse an fstring from a string, located at a certain position in the sourcecode.
 /// In case of errors, we will get the location and the error returned.
-pub fn parse_located_fstring(source: &str, location: Location) -> Result<Expr, FStringError> {
+pub fn parse_located_fstring(source: &str, location: Location) -> Result<Vec<Expr>, FStringError> {
     FStringParser::new(source, location, 0)
         .parse()
         .map_err(|error| FStringError { error, location })
@@ -308,7 +306,7 @@ pub fn parse_located_fstring(source: &str, location: Location) -> Result<Expr, F
 mod tests {
     use super::*;
 
-    fn parse_fstring(source: &str) -> Result<Expr, FStringErrorType> {
+    fn parse_fstring(source: &str) -> Result<Vec<Expr>, FStringErrorType> {
         FStringParser::new(source, Location::default(), 0).parse()
     }
 
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
index b2322b97..ac991599 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
@@ -2,62 +2,53 @@
 source: parser/src/fstring.rs
 expression: parse_ast
 ---
-Located {
-    location: Location {
-        row: 0,
-        column: 0,
+[
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "user=",
+            ),
+            kind: None,
+        },
     },
-    custom: (),
-    node: JoinedStr {
-        values: [
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: Constant {
-                    value: Str(
-                        "user=",
-                    ),
-                    kind: None,
-                },
-            },
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: Constant {
-                    value: Str(
-                        "",
-                    ),
-                    kind: None,
-                },
-            },
-            Located {
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "",
+            ),
+            kind: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: FormattedValue {
+            value: Located {
                 location: Location {
-                    row: 0,
-                    column: 0,
+                    row: 1,
+                    column: 2,
                 },
                 custom: (),
-                node: FormattedValue {
-                    value: Located {
-                        location: Location {
-                            row: 1,
-                            column: 2,
-                        },
-                        custom: (),
-                        node: Name {
-                            id: "user",
-                            ctx: Load,
-                        },
-                    },
-                    conversion: 114,
-                    format_spec: None,
+                node: Name {
+                    id: "user",
+                    ctx: Load,
                 },
             },
-        ],
+            conversion: 114,
+            format_spec: None,
+        },
     },
-}
+]
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
index 32d65816..6ebbcca2 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
@@ -2,136 +2,127 @@
 source: parser/src/fstring.rs
 expression: parse_ast
 ---
-Located {
-    location: Location {
-        row: 0,
-        column: 0,
+[
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "mix ",
+            ),
+            kind: None,
+        },
     },
-    custom: (),
-    node: JoinedStr {
-        values: [
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: Constant {
-                    value: Str(
-                        "mix ",
-                    ),
-                    kind: None,
-                },
-            },
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: Constant {
-                    value: Str(
-                        "user=",
-                    ),
-                    kind: None,
-                },
-            },
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: Constant {
-                    value: Str(
-                        "",
-                    ),
-                    kind: None,
-                },
-            },
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: FormattedValue {
-                    value: Located {
-                        location: Location {
-                            row: 1,
-                            column: 2,
-                        },
-                        custom: (),
-                        node: Name {
-                            id: "user",
-                            ctx: Load,
-                        },
-                    },
-                    conversion: 114,
-                    format_spec: None,
-                },
-            },
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: Constant {
-                    value: Str(
-                        " with text and ",
-                    ),
-                    kind: None,
-                },
-            },
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: Constant {
-                    value: Str(
-                        "second=",
-                    ),
-                    kind: None,
-                },
-            },
-            Located {
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "user=",
+            ),
+            kind: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "",
+            ),
+            kind: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: FormattedValue {
+            value: Located {
                 location: Location {
-                    row: 0,
-                    column: 0,
+                    row: 1,
+                    column: 2,
                 },
                 custom: (),
-                node: Constant {
-                    value: Str(
-                        "",
-                    ),
-                    kind: None,
+                node: Name {
+                    id: "user",
+                    ctx: Load,
                 },
             },
-            Located {
+            conversion: 114,
+            format_spec: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                " with text and ",
+            ),
+            kind: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "second=",
+            ),
+            kind: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "",
+            ),
+            kind: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: FormattedValue {
+            value: Located {
                 location: Location {
-                    row: 0,
-                    column: 0,
+                    row: 1,
+                    column: 2,
                 },
                 custom: (),
-                node: FormattedValue {
-                    value: Located {
-                        location: Location {
-                            row: 1,
-                            column: 2,
-                        },
-                        custom: (),
-                        node: Name {
-                            id: "second",
-                            ctx: Load,
-                        },
-                    },
-                    conversion: 114,
-                    format_spec: None,
+                node: Name {
+                    id: "second",
+                    ctx: Load,
                 },
             },
-        ],
+            conversion: 114,
+            format_spec: None,
+        },
     },
-}
+]
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
index 4ab56f07..71b86cb6 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
@@ -2,87 +2,78 @@
 source: parser/src/fstring.rs
 expression: parse_ast
 ---
-Located {
-    location: Location {
-        row: 0,
-        column: 0,
+[
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "user=",
+            ),
+            kind: None,
+        },
     },
-    custom: (),
-    node: JoinedStr {
-        values: [
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: Constant {
-                    value: Str(
-                        "user=",
-                    ),
-                    kind: None,
-                },
-            },
-            Located {
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "",
+            ),
+            kind: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: FormattedValue {
+            value: Located {
                 location: Location {
-                    row: 0,
-                    column: 0,
+                    row: 1,
+                    column: 2,
                 },
                 custom: (),
-                node: Constant {
-                    value: Str(
-                        "",
-                    ),
-                    kind: None,
+                node: Name {
+                    id: "user",
+                    ctx: Load,
                 },
             },
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: FormattedValue {
-                    value: Located {
-                        location: Location {
-                            row: 1,
-                            column: 2,
-                        },
-                        custom: (),
-                        node: Name {
-                            id: "user",
-                            ctx: Load,
-                        },
+            conversion: 0,
+            format_spec: Some(
+                Located {
+                    location: Location {
+                        row: 0,
+                        column: 0,
                     },
-                    conversion: 0,
-                    format_spec: Some(
-                        Located {
-                            location: Location {
-                                row: 0,
-                                column: 0,
+                    custom: (),
+                    node: JoinedStr {
+                        values: [
+                            Located {
+                                location: Location {
+                                    row: 0,
+                                    column: 0,
+                                },
+                                custom: (),
+                                node: Constant {
+                                    value: Str(
+                                        ">10",
+                                    ),
+                                    kind: None,
+                                },
                             },
-                            custom: (),
-                            node: JoinedStr {
-                                values: [
-                                    Located {
-                                        location: Location {
-                                            row: 0,
-                                            column: 0,
-                                        },
-                                        custom: (),
-                                        node: Constant {
-                                            value: Str(
-                                                ">10",
-                                            ),
-                                            kind: None,
-                                        },
-                                    },
-                                ],
-                            },
-                        },
-                    ),
+                        ],
+                    },
                 },
-            },
-        ],
+            ),
+        },
     },
-}
+]
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap
index 0da814cd..41567109 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap
@@ -1,15 +1,5 @@
 ---
 source: parser/src/fstring.rs
 expression: "parse_fstring(\"\").unwrap()"
-
 ---
-Located {
-    location: Location {
-        row: 0,
-        column: 0,
-    },
-    custom: (),
-    node: JoinedStr {
-        values: [],
-    },
-}
+[]
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
index 26af1d8e..3a05c326 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
@@ -2,71 +2,62 @@
 source: parser/src/fstring.rs
 expression: parse_ast
 ---
-Located {
-    location: Location {
-        row: 0,
-        column: 0,
-    },
-    custom: (),
-    node: JoinedStr {
-        values: [
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: FormattedValue {
-                    value: Located {
-                        location: Location {
-                            row: 1,
-                            column: 2,
-                        },
-                        custom: (),
-                        node: Name {
-                            id: "a",
-                            ctx: Load,
-                        },
-                    },
-                    conversion: 0,
-                    format_spec: None,
-                },
-            },
-            Located {
+[
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: FormattedValue {
+            value: Located {
                 location: Location {
-                    row: 0,
-                    column: 0,
+                    row: 1,
+                    column: 2,
                 },
                 custom: (),
-                node: FormattedValue {
-                    value: Located {
-                        location: Location {
-                            row: 1,
-                            column: 3,
-                        },
-                        custom: (),
-                        node: Name {
-                            id: "b",
-                            ctx: Load,
-                        },
-                    },
-                    conversion: 0,
-                    format_spec: None,
+                node: Name {
+                    id: "a",
+                    ctx: Load,
                 },
             },
-            Located {
+            conversion: 0,
+            format_spec: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: FormattedValue {
+            value: Located {
                 location: Location {
-                    row: 0,
-                    column: 0,
+                    row: 1,
+                    column: 3,
                 },
                 custom: (),
-                node: Constant {
-                    value: Str(
-                        "{foo}",
-                    ),
-                    kind: None,
+                node: Name {
+                    id: "b",
+                    ctx: Load,
                 },
             },
-        ],
+            conversion: 0,
+            format_spec: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "{foo}",
+            ),
+            kind: None,
+        },
     },
-}
+]
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
index 5a98ebdf..c7a6ddb2 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
@@ -2,65 +2,56 @@
 source: parser/src/fstring.rs
 expression: parse_ast
 ---
-Located {
-    location: Location {
-        row: 0,
-        column: 0,
-    },
-    custom: (),
-    node: JoinedStr {
-        values: [
-            Located {
+[
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: FormattedValue {
+            value: Located {
                 location: Location {
-                    row: 0,
-                    column: 0,
+                    row: 1,
+                    column: 5,
                 },
                 custom: (),
-                node: FormattedValue {
-                    value: Located {
+                node: Compare {
+                    left: Located {
                         location: Location {
                             row: 1,
-                            column: 5,
+                            column: 2,
                         },
                         custom: (),
-                        node: Compare {
-                            left: Located {
-                                location: Location {
-                                    row: 1,
-                                    column: 2,
-                                },
-                                custom: (),
-                                node: Constant {
-                                    value: Int(
-                                        42,
-                                    ),
-                                    kind: None,
-                                },
-                            },
-                            ops: [
-                                Eq,
-                            ],
-                            comparators: [
-                                Located {
-                                    location: Location {
-                                        row: 1,
-                                        column: 8,
-                                    },
-                                    custom: (),
-                                    node: Constant {
-                                        value: Int(
-                                            42,
-                                        ),
-                                        kind: None,
-                                    },
-                                },
-                            ],
+                        node: Constant {
+                            value: Int(
+                                42,
+                            ),
+                            kind: None,
                         },
                     },
-                    conversion: 0,
-                    format_spec: None,
+                    ops: [
+                        Eq,
+                    ],
+                    comparators: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 8,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    42,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
                 },
             },
-        ],
+            conversion: 0,
+            format_spec: None,
+        },
     },
-}
+]
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
index 3bf74c02..e71cb7ce 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
@@ -2,118 +2,52 @@
 source: parser/src/fstring.rs
 expression: parse_ast
 ---
-Located {
-    location: Location {
-        row: 0,
-        column: 0,
-    },
-    custom: (),
-    node: JoinedStr {
-        values: [
-            Located {
+[
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: FormattedValue {
+            value: Located {
                 location: Location {
-                    row: 0,
-                    column: 0,
+                    row: 1,
+                    column: 2,
                 },
                 custom: (),
-                node: FormattedValue {
-                    value: Located {
-                        location: Location {
-                            row: 1,
-                            column: 2,
-                        },
-                        custom: (),
-                        node: Name {
-                            id: "foo",
-                            ctx: Load,
-                        },
+                node: Name {
+                    id: "foo",
+                    ctx: Load,
+                },
+            },
+            conversion: 0,
+            format_spec: Some(
+                Located {
+                    location: Location {
+                        row: 0,
+                        column: 0,
                     },
-                    conversion: 0,
-                    format_spec: Some(
-                        Located {
-                            location: Location {
-                                row: 0,
-                                column: 0,
+                    custom: (),
+                    node: JoinedStr {
+                        values: [
+                            Located {
+                                location: Location {
+                                    row: 0,
+                                    column: 0,
+                                },
+                                custom: (),
+                                node: Constant {
+                                    value: Str(
+                                        "{ spec}",
+                                    ),
+                                    kind: None,
+                                },
                             },
-                            custom: (),
-                            node: JoinedStr {
-                                values: [
-                                    Located {
-                                        location: Location {
-                                            row: 0,
-                                            column: 0,
-                                        },
-                                        custom: (),
-                                        node: Constant {
-                                            value: Str(
-                                                "",
-                                            ),
-                                            kind: None,
-                                        },
-                                    },
-                                    Located {
-                                        location: Location {
-                                            row: 0,
-                                            column: 0,
-                                        },
-                                        custom: (),
-                                        node: FormattedValue {
-                                            value: Located {
-                                                location: Location {
-                                                    row: 0,
-                                                    column: 0,
-                                                },
-                                                custom: (),
-                                                node: JoinedStr {
-                                                    values: [
-                                                        Located {
-                                                            location: Location {
-                                                                row: 0,
-                                                                column: 0,
-                                                            },
-                                                            custom: (),
-                                                            node: FormattedValue {
-                                                                value: Located {
-                                                                    location: Location {
-                                                                        row: 1,
-                                                                        column: 3,
-                                                                    },
-                                                                    custom: (),
-                                                                    node: Name {
-                                                                        id: "spec",
-                                                                        ctx: Load,
-                                                                    },
-                                                                },
-                                                                conversion: 0,
-                                                                format_spec: None,
-                                                            },
-                                                        },
-                                                    ],
-                                                },
-                                            },
-                                            conversion: 0,
-                                            format_spec: None,
-                                        },
-                                    },
-                                    Located {
-                                        location: Location {
-                                            row: 0,
-                                            column: 0,
-                                        },
-                                        custom: (),
-                                        node: Constant {
-                                            value: Str(
-                                                "",
-                                            ),
-                                            kind: None,
-                                        },
-                                    },
-                                ],
-                            },
-                        },
-                    ),
+                        ],
+                    },
                 },
-            },
-        ],
+            ),
+        },
     },
-}
+]
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
index 655bf1b3..d3ca5d39 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
@@ -2,65 +2,56 @@
 source: parser/src/fstring.rs
 expression: parse_ast
 ---
-Located {
-    location: Location {
-        row: 0,
-        column: 0,
-    },
-    custom: (),
-    node: JoinedStr {
-        values: [
-            Located {
+[
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: FormattedValue {
+            value: Located {
                 location: Location {
-                    row: 0,
-                    column: 0,
+                    row: 1,
+                    column: 4,
                 },
                 custom: (),
-                node: FormattedValue {
-                    value: Located {
+                node: Compare {
+                    left: Located {
                         location: Location {
                             row: 1,
-                            column: 4,
+                            column: 2,
                         },
                         custom: (),
-                        node: Compare {
-                            left: Located {
-                                location: Location {
-                                    row: 1,
-                                    column: 2,
-                                },
-                                custom: (),
-                                node: Constant {
-                                    value: Int(
-                                        1,
-                                    ),
-                                    kind: None,
-                                },
-                            },
-                            ops: [
-                                NotEq,
-                            ],
-                            comparators: [
-                                Located {
-                                    location: Location {
-                                        row: 1,
-                                        column: 7,
-                                    },
-                                    custom: (),
-                                    node: Constant {
-                                        value: Int(
-                                            2,
-                                        ),
-                                        kind: None,
-                                    },
-                                },
-                            ],
+                        node: Constant {
+                            value: Int(
+                                1,
+                            ),
+                            kind: None,
                         },
                     },
-                    conversion: 0,
-                    format_spec: None,
+                    ops: [
+                        NotEq,
+                    ],
+                    comparators: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 7,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
                 },
             },
-        ],
+            conversion: 0,
+            format_spec: None,
+        },
     },
-}
+]
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
index 8950ebbe..a5760e2d 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
@@ -2,61 +2,52 @@
 source: parser/src/fstring.rs
 expression: parse_ast
 ---
-Located {
-    location: Location {
-        row: 0,
-        column: 0,
-    },
-    custom: (),
-    node: JoinedStr {
-        values: [
-            Located {
+[
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: FormattedValue {
+            value: Located {
                 location: Location {
-                    row: 0,
-                    column: 0,
+                    row: 1,
+                    column: 2,
                 },
                 custom: (),
-                node: FormattedValue {
-                    value: Located {
-                        location: Location {
-                            row: 1,
-                            column: 2,
-                        },
-                        custom: (),
-                        node: Name {
-                            id: "foo",
-                            ctx: Load,
-                        },
+                node: Name {
+                    id: "foo",
+                    ctx: Load,
+                },
+            },
+            conversion: 0,
+            format_spec: Some(
+                Located {
+                    location: Location {
+                        row: 0,
+                        column: 0,
                     },
-                    conversion: 0,
-                    format_spec: Some(
-                        Located {
-                            location: Location {
-                                row: 0,
-                                column: 0,
+                    custom: (),
+                    node: JoinedStr {
+                        values: [
+                            Located {
+                                location: Location {
+                                    row: 0,
+                                    column: 0,
+                                },
+                                custom: (),
+                                node: Constant {
+                                    value: Str(
+                                        "spec",
+                                    ),
+                                    kind: None,
+                                },
                             },
-                            custom: (),
-                            node: JoinedStr {
-                                values: [
-                                    Located {
-                                        location: Location {
-                                            row: 0,
-                                            column: 0,
-                                        },
-                                        custom: (),
-                                        node: Constant {
-                                            value: Str(
-                                                "spec",
-                                            ),
-                                            kind: None,
-                                        },
-                                    },
-                                ],
-                            },
-                        },
-                    ),
+                        ],
+                    },
                 },
-            },
-        ],
+            ),
+        },
     },
-}
+]
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
index 33f62eed..0e92a121 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
@@ -2,62 +2,53 @@
 source: parser/src/fstring.rs
 expression: parse_ast
 ---
-Located {
-    location: Location {
-        row: 0,
-        column: 0,
+[
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "x   =",
+            ),
+            kind: None,
+        },
     },
-    custom: (),
-    node: JoinedStr {
-        values: [
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: Constant {
-                    value: Str(
-                        "x   =",
-                    ),
-                    kind: None,
-                },
-            },
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: Constant {
-                    value: Str(
-                        "",
-                    ),
-                    kind: None,
-                },
-            },
-            Located {
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "",
+            ),
+            kind: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: FormattedValue {
+            value: Located {
                 location: Location {
-                    row: 0,
-                    column: 0,
+                    row: 1,
+                    column: 2,
                 },
                 custom: (),
-                node: FormattedValue {
-                    value: Located {
-                        location: Location {
-                            row: 1,
-                            column: 2,
-                        },
-                        custom: (),
-                        node: Name {
-                            id: "x",
-                            ctx: Load,
-                        },
-                    },
-                    conversion: 114,
-                    format_spec: None,
+                node: Name {
+                    id: "x",
+                    ctx: Load,
                 },
             },
-        ],
+            conversion: 114,
+            format_spec: None,
+        },
     },
-}
+]
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
index deb7b02b..22bb47fc 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
@@ -2,62 +2,53 @@
 source: parser/src/fstring.rs
 expression: parse_ast
 ---
-Located {
-    location: Location {
-        row: 0,
-        column: 0,
+[
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "x=",
+            ),
+            kind: None,
+        },
     },
-    custom: (),
-    node: JoinedStr {
-        values: [
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: Constant {
-                    value: Str(
-                        "x=",
-                    ),
-                    kind: None,
-                },
-            },
-            Located {
-                location: Location {
-                    row: 0,
-                    column: 0,
-                },
-                custom: (),
-                node: Constant {
-                    value: Str(
-                        "   ",
-                    ),
-                    kind: None,
-                },
-            },
-            Located {
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: Constant {
+            value: Str(
+                "   ",
+            ),
+            kind: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: FormattedValue {
+            value: Located {
                 location: Location {
-                    row: 0,
-                    column: 0,
+                    row: 1,
+                    column: 2,
                 },
                 custom: (),
-                node: FormattedValue {
-                    value: Located {
-                        location: Location {
-                            row: 1,
-                            column: 2,
-                        },
-                        custom: (),
-                        node: Name {
-                            id: "x",
-                            ctx: Load,
-                        },
-                    },
-                    conversion: 114,
-                    format_spec: None,
+                node: Name {
+                    id: "x",
+                    ctx: Load,
                 },
             },
-        ],
+            conversion: 114,
+            format_spec: None,
+        },
     },
-}
+]
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
index 37e0d05a..a9637393 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
@@ -2,35 +2,26 @@
 source: parser/src/fstring.rs
 expression: parse_ast
 ---
-Located {
-    location: Location {
-        row: 0,
-        column: 0,
-    },
-    custom: (),
-    node: JoinedStr {
-        values: [
-            Located {
+[
+    Located {
+        location: Location {
+            row: 0,
+            column: 0,
+        },
+        custom: (),
+        node: FormattedValue {
+            value: Located {
                 location: Location {
-                    row: 0,
-                    column: 0,
+                    row: 1,
+                    column: 2,
                 },
                 custom: (),
-                node: FormattedValue {
-                    value: Located {
-                        location: Location {
-                            row: 1,
-                            column: 2,
-                        },
-                        custom: (),
-                        node: Yield {
-                            value: None,
-                        },
-                    },
-                    conversion: 0,
-                    format_spec: None,
+                node: Yield {
+                    value: None,
                 },
             },
-        ],
+            conversion: 0,
+            format_spec: None,
+        },
     },
-}
+]
diff --git a/parser/src/string.rs b/parser/src/string.rs
index 02a55145..ac4d2221 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -34,19 +34,10 @@ pub fn parse_strings(values: Vec<(Location, (String, StringKind))>) -> Result<Ex
             StringKind::Normal | StringKind::U => current.push(string),
             StringKind::F => {
                 has_fstring = true;
-                let values = if let ExprKind::JoinedStr { values } =
-                    parse_located_fstring(&string, location)
-                        .map_err(|e| LexicalError {
-                            location,
-                            error: LexicalErrorType::FStringError(e.error),
-                        })?
-                        .node
-                {
-                    values
-                } else {
-                    unreachable!("parse_located_fstring returned a non-JoinedStr.")
-                };
-                for value in values {
+                for value in parse_located_fstring(&string, location).map_err(|e| LexicalError {
+                    location,
+                    error: LexicalErrorType::FStringError(e.error),
+                })? {
                     match value.node {
                         ExprKind::FormattedValue { .. } => {
                             if !current.is_empty() {

From 147187d18dd6a73ea25d460af407051b6a227ae7 Mon Sep 17 00:00:00 2001
From: Charles Marsh <charlie@springdisc.com>
Date: Sun, 21 Aug 2022 16:55:06 -0400
Subject: [PATCH 252/459] Fix f-string regression

---
 parser/src/fstring.rs                         | 18 +++++++++-------
 ...s__fstring_parse_selfdocumenting_base.snap |  2 +-
 ...tring_parse_selfdocumenting_base_more.snap |  2 +-
 ..._fstring_parse_selfdocumenting_format.snap |  2 +-
 ...__fstring__tests__parse_empty_fstring.snap |  2 +-
 ...parser__fstring__tests__parse_fstring.snap |  2 +-
 ..._fstring__tests__parse_fstring_equals.snap |  2 +-
 ...ing__tests__parse_fstring_nested_spec.snap | 21 +++++++++++++------
 ...ring__tests__parse_fstring_not_equals.snap |  2 +-
 ..._tests__parse_fstring_not_nested_spec.snap |  2 +-
 ...sts__parse_fstring_selfdoc_prec_space.snap |  2 +-
 ..._parse_fstring_selfdoc_trailing_space.snap |  2 +-
 ...ring__tests__parse_fstring_yield_expr.snap |  2 +-
 13 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index 0d5e2605..2463e7be 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -4,6 +4,7 @@ use crate::{
     error::{FStringError, FStringErrorType, ParseError},
     parser::parse_expression,
 };
+use itertools::Itertools;
 use std::{iter, mem, str};
 
 struct FStringParser<'a> {
@@ -105,13 +106,16 @@ impl<'a> FStringParser<'a> {
                                 nested -= 1;
                                 if nested == 0 {
                                     formatted_value_piece.push(next);
-                                    spec_constructor.extend(
-                                        FStringParser::new(
-                                            &format!("{{{}}}", formatted_value_piece),
-                                            Location::default(),
-                                            &self.recurse_lvl + 1,
-                                        )
-                                        .parse()?,
+                                    let values = FStringParser::new(
+                                        &formatted_value_piece,
+                                        Location::default(),
+                                        &self.recurse_lvl + 1,
+                                    )
+                                    .parse()?;
+                                    spec_constructor.push(values
+                                        .into_iter()
+                                        .exactly_one()
+                                        .expect("Expected formatted value to produce exactly one expression.")
                                     );
                                     formatted_value_piece.clear();
                                 } else {
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
index ac991599..14e7003f 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/fstring.rs
+source: compiler/parser/src/fstring.rs
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
index 6ebbcca2..0df5ab2a 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/fstring.rs
+source: compiler/parser/src/fstring.rs
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
index 71b86cb6..f5a518d7 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/fstring.rs
+source: compiler/parser/src/fstring.rs
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap
index 41567109..e561a7f9 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/fstring.rs
+source: compiler/parser/src/fstring.rs
 expression: "parse_fstring(\"\").unwrap()"
 ---
 []
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
index 3a05c326..7210ed34 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/fstring.rs
+source: compiler/parser/src/fstring.rs
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
index c7a6ddb2..f9d6c4d2 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/fstring.rs
+source: compiler/parser/src/fstring.rs
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
index e71cb7ce..a0262f1a 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/fstring.rs
+source: compiler/parser/src/fstring.rs
 expression: parse_ast
 ---
 [
@@ -37,11 +37,20 @@ expression: parse_ast
                                     column: 0,
                                 },
                                 custom: (),
-                                node: Constant {
-                                    value: Str(
-                                        "{ spec}",
-                                    ),
-                                    kind: None,
+                                node: FormattedValue {
+                                    value: Located {
+                                        location: Location {
+                                            row: 1,
+                                            column: 3,
+                                        },
+                                        custom: (),
+                                        node: Name {
+                                            id: "spec",
+                                            ctx: Load,
+                                        },
+                                    },
+                                    conversion: 0,
+                                    format_spec: None,
                                 },
                             },
                         ],
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
index d3ca5d39..f9223a59 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/fstring.rs
+source: compiler/parser/src/fstring.rs
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
index a5760e2d..e15ac253 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/fstring.rs
+source: compiler/parser/src/fstring.rs
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
index 0e92a121..ae998f0a 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/fstring.rs
+source: compiler/parser/src/fstring.rs
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
index 22bb47fc..317bded0 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/fstring.rs
+source: compiler/parser/src/fstring.rs
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
index a9637393..cb7d7208 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/fstring.rs
+source: compiler/parser/src/fstring.rs
 expression: parse_ast
 ---
 [

From 9c229ebb99a13b5cff21d8ce9aedb4cab0860bab Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 05:19:36 +0900
Subject: [PATCH 253/459] Refactor symboltables and _ast to use codegen when
 possible

---
 codegen/src/compile.rs     | 14 +++++++-------
 codegen/src/lib.rs         |  2 ++
 codegen/src/symboltable.rs | 24 ++++++++++++------------
 src/lib.rs                 |  9 +++------
 4 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 895e8690..fed7df4b 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -8,7 +8,7 @@
 use crate::{
     error::{CompileError, CompileErrorType},
     ir,
-    symboltable::{self, make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable},
+    symboltable::{self, SymbolScope, SymbolTable},
     IndexSet,
 };
 use itertools::Itertools;
@@ -149,7 +149,7 @@ pub fn compile_program(
         ast,
         source_path,
         opts,
-        make_symbol_table,
+        SymbolTable::scan_program,
         Compiler::compile_program,
     )
 }
@@ -164,7 +164,7 @@ pub fn compile_program_single(
         ast,
         source_path,
         opts,
-        make_symbol_table,
+        SymbolTable::scan_program,
         Compiler::compile_program_single,
     )
 }
@@ -178,7 +178,7 @@ pub fn compile_block_expression(
         ast,
         source_path,
         opts,
-        make_symbol_table,
+        SymbolTable::scan_program,
         Compiler::compile_block_expr,
     )
 }
@@ -192,7 +192,7 @@ pub fn compile_expression(
         ast,
         source_path,
         opts,
-        make_symbol_table_expr,
+        SymbolTable::scan_expr,
         Compiler::compile_eval,
     )
 }
@@ -2697,7 +2697,7 @@ fn compile_constant(value: &ast::Constant) -> ConstantData {
 #[cfg(test)]
 mod tests {
     use super::{CompileOpts, Compiler};
-    use crate::symboltable::make_symbol_table;
+    use crate::symboltable::SymbolTable;
     use rustpython_bytecode::CodeObject;
     use rustpython_parser::parser;
 
@@ -2708,7 +2708,7 @@ mod tests {
             "<module>".to_owned(),
         );
         let ast = parser::parse_program(source).unwrap();
-        let symbol_scope = make_symbol_table(&ast).unwrap();
+        let symbol_scope = SymbolTable::scan_program(&ast).unwrap();
         compiler.compile_program(&ast, symbol_scope).unwrap();
         compiler.pop_code_object()
     }
diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs
index 3e3aca8f..4bef9cc9 100644
--- a/codegen/src/lib.rs
+++ b/codegen/src/lib.rs
@@ -13,3 +13,5 @@ pub mod error;
 pub mod ir;
 pub mod mode;
 pub mod symboltable;
+
+pub use compile::{CompileOpts, Mode};
diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 6dad4f23..5c9235d8 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -14,18 +14,6 @@ use crate::{
 use rustpython_ast::{self as ast, Location};
 use std::{borrow::Cow, fmt};
 
-pub fn make_symbol_table(program: &[ast::Stmt]) -> SymbolTableResult<SymbolTable> {
-    let mut builder = SymbolTableBuilder::new();
-    builder.scan_statements(program)?;
-    builder.finish()
-}
-
-pub fn make_symbol_table_expr(expr: &ast::Expr) -> SymbolTableResult<SymbolTable> {
-    let mut builder = SymbolTableBuilder::new();
-    builder.scan_expression(expr, ExpressionContext::Load)?;
-    builder.finish()
-}
-
 /// Captures all symbols in the current scope, and has a list of subscopes in this scope.
 #[derive(Clone)]
 pub struct SymbolTable {
@@ -60,6 +48,18 @@ impl SymbolTable {
             sub_tables: vec![],
         }
     }
+
+    pub fn scan_program(program: &[ast::Stmt]) -> SymbolTableResult<Self> {
+        let mut builder = SymbolTableBuilder::new();
+        builder.scan_statements(program)?;
+        builder.finish()
+    }
+
+    pub fn scan_expr(expr: &ast::Expr) -> SymbolTableResult<Self> {
+        let mut builder = SymbolTableBuilder::new();
+        builder.scan_expression(expr, ExpressionContext::Load)?;
+        builder.finish()
+    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
diff --git a/src/lib.rs b/src/lib.rs
index 1a659a88..36491d65 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -6,9 +6,6 @@ use rustpython_parser::{
 };
 use std::fmt;
 
-pub use compile::{CompileOpts, Mode};
-pub use symboltable::{Symbol, SymbolScope, SymbolTable, SymbolTableType};
-
 #[derive(Debug, thiserror::Error)]
 pub enum CompileErrorType {
     #[error(transparent)]
@@ -76,7 +73,7 @@ pub fn compile(
     source: &str,
     mode: compile::Mode,
     source_path: String,
-    opts: CompileOpts,
+    opts: compile::CompileOpts,
 ) -> Result<CodeObject, CompileError> {
     let parser_mode = match mode {
         compile::Mode::Exec => parser::Mode::Module,
@@ -105,11 +102,11 @@ pub fn compile_symtable(
     let res = match mode {
         compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
             let ast = parser::parse_program(source).map_err(parse_err)?;
-            symboltable::make_symbol_table(&ast)
+            symboltable::SymbolTable::scan_program(&ast)
         }
         compile::Mode::Eval => {
             let expr = parser::parse_expression(source).map_err(parse_err)?;
-            symboltable::make_symbol_table_expr(&expr)
+            symboltable::SymbolTable::scan_expr(&expr)
         }
     };
     res.map_err(|e| CompileError::from_symtable(e, source, source_path.to_owned()))

From 3690272256a62c792a68a41c8b4ad91798dbf851 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 06:09:18 +0900
Subject: [PATCH 254/459] Refactor Mode and partial parser/codegen for
 eval/exec

---
 bytecode/src/lib.rs               | 3 +++
 {codegen => bytecode}/src/mode.rs | 0
 codegen/src/compile.rs            | 2 +-
 codegen/src/lib.rs                | 1 -
 src/lib.rs                        | 3 +++
 5 files changed, 7 insertions(+), 2 deletions(-)
 rename {codegen => bytecode}/src/mode.rs (100%)

diff --git a/bytecode/src/lib.rs b/bytecode/src/lib.rs
index f4f3f306..c53697bc 100644
--- a/bytecode/src/lib.rs
+++ b/bytecode/src/lib.rs
@@ -4,6 +4,9 @@
 #![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
 #![doc(html_root_url = "https://docs.rs/rustpython-bytecode/")]
 
+mod mode;
+pub use mode::Mode;
+
 use bitflags::bitflags;
 use bstr::ByteSlice;
 use itertools::Itertools;
diff --git a/codegen/src/mode.rs b/bytecode/src/mode.rs
similarity index 100%
rename from codegen/src/mode.rs
rename to bytecode/src/mode.rs
diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index fed7df4b..6eecd2c0 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -18,7 +18,7 @@ use rustpython_ast as ast;
 use rustpython_bytecode::{self as bytecode, CodeObject, ConstantData, Instruction};
 use std::borrow::Cow;
 
-pub use crate::mode::Mode;
+pub use rustpython_bytecode::Mode;
 
 type CompileResult<T> = Result<T, CompileError>;
 
diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs
index 4bef9cc9..9114a049 100644
--- a/codegen/src/lib.rs
+++ b/codegen/src/lib.rs
@@ -11,7 +11,6 @@ type IndexSet<T> = indexmap::IndexSet<T, ahash::RandomState>;
 pub mod compile;
 pub mod error;
 pub mod ir;
-pub mod mode;
 pub mod symboltable;
 
 pub use compile::{CompileOpts, Mode};
diff --git a/src/lib.rs b/src/lib.rs
index 36491d65..f09af260 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -6,6 +6,9 @@ use rustpython_parser::{
 };
 use std::fmt;
 
+pub use rustpython_bytecode::Mode;
+pub use rustpython_codegen::compile::CompileOpts;
+
 #[derive(Debug, thiserror::Error)]
 pub enum CompileErrorType {
     #[error(transparent)]

From cf10e800a60d244b1c895acfe2f291f1df44a30d Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 06:30:06 +0900
Subject: [PATCH 255/459] codegen::{CompileError -> CodegenError}

---
 codegen/src/compile.rs     | 66 +++++++++++++++++++-------------------
 codegen/src/error.rs       | 55 +++++++++++++++----------------
 codegen/src/symboltable.rs |  8 ++---
 src/lib.rs                 | 14 ++++----
 4 files changed, 72 insertions(+), 71 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 6eecd2c0..f7dd2727 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -6,7 +6,7 @@
 //!   <https://github.com/micropython/micropython/blob/master/py/compile.c>
 
 use crate::{
-    error::{CompileError, CompileErrorType},
+    error::{CodegenError, CodegenErrorType},
     ir,
     symboltable::{self, SymbolScope, SymbolTable},
     IndexSet,
@@ -20,7 +20,7 @@ use std::borrow::Cow;
 
 pub use rustpython_bytecode::Mode;
 
-type CompileResult<T> = Result<T, CompileError>;
+type CompileResult<T> = Result<T, CodegenError>;
 
 #[derive(PartialEq, Eq, Clone, Copy)]
 enum NameUsage {
@@ -129,7 +129,7 @@ fn compile_impl<Ast: ?Sized>(
 ) -> CompileResult<CodeObject> {
     let symbol_table = match make_symbol_table(ast) {
         Ok(x) => x,
-        Err(e) => return Err(e.into_compile_error(source_path)),
+        Err(e) => return Err(e.into_codegen_error(source_path)),
     };
 
     let mut compiler = Compiler::new(opts, source_path, "<module>".to_owned());
@@ -234,11 +234,11 @@ impl Compiler {
         }
     }
 
-    fn error(&self, error: CompileErrorType) -> CompileError {
+    fn error(&self, error: CodegenErrorType) -> CodegenError {
         self.error_loc(error, self.current_source_location)
     }
-    fn error_loc(&self, error: CompileErrorType, location: ast::Location) -> CompileError {
-        CompileError {
+    fn error_loc(&self, error: CodegenErrorType, location: ast::Location) -> CodegenError {
+        CodegenError {
             error,
             location,
             source_path: self.source_path.clone(),
@@ -455,7 +455,7 @@ impl Compiler {
             NameUsage::Delete if is_forbidden_name(name) => "cannot delete",
             _ => return Ok(()),
         };
-        Err(self.error(CompileErrorType::SyntaxError(format!("{} {}", msg, name))))
+        Err(self.error(CodegenErrorType::SyntaxError(format!("{} {}", msg, name))))
     }
 
     fn compile_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> {
@@ -580,7 +580,7 @@ impl Compiler {
                 let from_list = if import_star {
                     if self.ctx.in_func() {
                         return Err(self
-                            .error_loc(CompileErrorType::FunctionImportStar, statement.location));
+                            .error_loc(CodegenErrorType::FunctionImportStar, statement.location));
                     }
                     vec![ConstantData::Str {
                         value: "*".to_owned(),
@@ -769,7 +769,7 @@ impl Compiler {
                     self.emit(Instruction::Break { target: end });
                 }
                 None => {
-                    return Err(self.error_loc(CompileErrorType::InvalidBreak, statement.location));
+                    return Err(self.error_loc(CodegenErrorType::InvalidBreak, statement.location));
                 }
             },
             Continue => match self.ctx.loop_data {
@@ -778,13 +778,13 @@ impl Compiler {
                 }
                 None => {
                     return Err(
-                        self.error_loc(CompileErrorType::InvalidContinue, statement.location)
+                        self.error_loc(CodegenErrorType::InvalidContinue, statement.location)
                     );
                 }
             },
             Return { value } => {
                 if !self.ctx.in_func() {
-                    return Err(self.error_loc(CompileErrorType::InvalidReturn, statement.location));
+                    return Err(self.error_loc(CodegenErrorType::InvalidReturn, statement.location));
                 }
                 match value {
                     Some(v) => {
@@ -795,7 +795,7 @@ impl Compiler {
                                 .contains(bytecode::CodeFlags::IS_GENERATOR)
                         {
                             return Err(self.error_loc(
-                                CompileErrorType::AsyncReturnValue,
+                                CodegenErrorType::AsyncReturnValue,
                                 statement.location,
                             ));
                         }
@@ -857,9 +857,9 @@ impl Compiler {
                 }
             }
             ast::ExprKind::BinOp { .. } | ast::ExprKind::UnaryOp { .. } => {
-                return Err(self.error(CompileErrorType::Delete("expression")))
+                return Err(self.error(CodegenErrorType::Delete("expression")))
             }
-            _ => return Err(self.error(CompileErrorType::Delete(expression.node.name()))),
+            _ => return Err(self.error(CodegenErrorType::Delete(expression.node.name()))),
         }
         Ok(())
     }
@@ -922,7 +922,7 @@ impl Compiler {
             .chain(&args.kwonlyargs);
         for name in args_iter {
             if Compiler::is_forbidden_arg_name(&name.node.arg) {
-                return Err(self.error(CompileErrorType::SyntaxError(format!(
+                return Err(self.error(CodegenErrorType::SyntaxError(format!(
                     "cannot assign to {}",
                     &name.node.arg
                 ))));
@@ -1401,7 +1401,7 @@ impl Compiler {
         let (item, items) = if let Some(parts) = items.split_first() {
             parts
         } else {
-            return Err(self.error(CompileErrorType::EmptyWithItems));
+            return Err(self.error(CodegenErrorType::EmptyWithItems));
         };
 
         let final_block = {
@@ -1433,7 +1433,7 @@ impl Compiler {
 
         if items.is_empty() {
             if body.is_empty() {
-                return Err(self.error(CompileErrorType::EmptyWithBody));
+                return Err(self.error(CodegenErrorType::EmptyWithBody));
             }
             self.compile_statements(body)?;
         } else {
@@ -1529,7 +1529,7 @@ impl Compiler {
     ) -> CompileResult<()> {
         eprintln!("match subject: {subject:?}");
         eprintln!("match cases: {cases:?}");
-        Err(self.error(CompileErrorType::NotImplementedYet))
+        Err(self.error(CodegenErrorType::NotImplementedYet))
     }
 
     fn compile_chained_comparison(
@@ -1699,7 +1699,7 @@ impl Compiler {
                 for (i, element) in elts.iter().enumerate() {
                     if let ast::ExprKind::Starred { .. } = &element.node {
                         if seen_star {
-                            return Err(self.error(CompileErrorType::MultipleStarArgs));
+                            return Err(self.error(CodegenErrorType::MultipleStarArgs));
                         } else {
                             seen_star = true;
                             let before = i;
@@ -1707,7 +1707,7 @@ impl Compiler {
                             let (before, after) = (|| Some((before.to_u8()?, after.to_u8()?)))()
                                 .ok_or_else(|| {
                                     self.error_loc(
-                                        CompileErrorType::TooManyStarUnpack,
+                                        CodegenErrorType::TooManyStarUnpack,
                                         target.location,
                                     )
                                 })?;
@@ -1732,10 +1732,10 @@ impl Compiler {
             }
             _ => {
                 return Err(self.error(match target.node {
-                    ast::ExprKind::Starred { .. } => CompileErrorType::SyntaxError(
+                    ast::ExprKind::Starred { .. } => CodegenErrorType::SyntaxError(
                         "starred assignment target must be in a list or tuple".to_owned(),
                     ),
-                    _ => CompileErrorType::Assign(target.node.name()),
+                    _ => CodegenErrorType::Assign(target.node.name()),
                 }));
             }
         }
@@ -1776,7 +1776,7 @@ impl Compiler {
                 AugAssignKind::Attr { idx }
             }
             _ => {
-                return Err(self.error(CompileErrorType::Assign(target.node.name())));
+                return Err(self.error(CodegenErrorType::Assign(target.node.name())));
             }
         };
 
@@ -2049,7 +2049,7 @@ impl Compiler {
             }
             Yield { value } => {
                 if !self.ctx.in_func() {
-                    return Err(self.error(CompileErrorType::InvalidYield));
+                    return Err(self.error(CodegenErrorType::InvalidYield));
                 }
                 self.mark_generator();
                 match value {
@@ -2060,7 +2060,7 @@ impl Compiler {
             }
             Await { value } => {
                 if self.ctx.func != FunctionContext::AsyncFunction {
-                    return Err(self.error(CompileErrorType::InvalidAwait));
+                    return Err(self.error(CodegenErrorType::InvalidAwait));
                 }
                 self.compile_expression(value)?;
                 self.emit(Instruction::GetAwaitable);
@@ -2070,10 +2070,10 @@ impl Compiler {
             YieldFrom { value } => {
                 match self.ctx.func {
                     FunctionContext::NoFunction => {
-                        return Err(self.error(CompileErrorType::InvalidYieldFrom));
+                        return Err(self.error(CodegenErrorType::InvalidYieldFrom));
                     }
                     FunctionContext::AsyncFunction => {
-                        return Err(self.error(CompileErrorType::AsyncYieldFrom));
+                        return Err(self.error(CodegenErrorType::AsyncYieldFrom));
                     }
                     FunctionContext::Function => {}
                 }
@@ -2208,7 +2208,7 @@ impl Compiler {
                 })?;
             }
             Starred { .. } => {
-                return Err(self.error(CompileErrorType::InvalidStarExpr));
+                return Err(self.error(CodegenErrorType::InvalidStarExpr));
             }
             IfExp { test, body, orelse } => {
                 let else_block = self.new_block();
@@ -2417,8 +2417,8 @@ impl Compiler {
 
     fn compile_comprehension_element(&mut self, element: &ast::Expr) -> CompileResult<()> {
         self.compile_expression(element).map_err(|e| {
-            if let CompileErrorType::InvalidStarExpr = e.error {
-                self.error(CompileErrorType::SyntaxError(
+            if let CodegenErrorType::InvalidStarExpr = e.error {
+                self.error(CodegenErrorType::SyntaxError(
                     "iterable unpacking cannot be used in comprehension".to_owned(),
                 ))
             } else {
@@ -2547,9 +2547,9 @@ impl Compiler {
         Ok(())
     }
 
-    fn compile_future_features(&mut self, features: &[ast::Alias]) -> Result<(), CompileError> {
+    fn compile_future_features(&mut self, features: &[ast::Alias]) -> Result<(), CodegenError> {
         if self.done_with_future_stmts {
-            return Err(self.error(CompileErrorType::InvalidFuturePlacement));
+            return Err(self.error(CodegenErrorType::InvalidFuturePlacement));
         }
         for feature in features {
             match &*feature.node.name {
@@ -2559,7 +2559,7 @@ impl Compiler {
                 // "generator_stop" => {}
                 "annotations" => self.future_annotations = true,
                 other => {
-                    return Err(self.error(CompileErrorType::InvalidFutureFeature(other.to_owned())))
+                    return Err(self.error(CodegenErrorType::InvalidFutureFeature(other.to_owned())))
                 }
             }
         }
diff --git a/codegen/src/error.rs b/codegen/src/error.rs
index 2e125b01..60444a20 100644
--- a/codegen/src/error.rs
+++ b/codegen/src/error.rs
@@ -3,15 +3,15 @@ use rustpython_ast::Location;
 use std::{error::Error, fmt};
 
 #[derive(Debug)]
-pub struct CompileError {
-    pub error: CompileErrorType,
+pub struct CodegenError {
+    pub error: CodegenErrorType,
     pub location: Location,
     pub source_path: String,
 }
 
 #[derive(Debug)]
 #[non_exhaustive]
-pub enum CompileErrorType {
+pub enum CodegenErrorType {
     /// Invalid assignment, cannot store value in target.
     Assign(&'static str),
     /// Invalid delete
@@ -40,61 +40,62 @@ pub enum CompileErrorType {
     NotImplementedYet, // RustPython marker for unimplemented features
 }
 
-impl fmt::Display for CompileErrorType {
+impl fmt::Display for CodegenErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        use CodegenErrorType::*;
         match self {
-            CompileErrorType::Assign(target) => write!(f, "cannot assign to {}", target),
-            CompileErrorType::Delete(target) => write!(f, "cannot delete {}", target),
-            CompileErrorType::SyntaxError(err) => write!(f, "{}", err.as_str()),
-            CompileErrorType::MultipleStarArgs => {
+            Assign(target) => write!(f, "cannot assign to {}", target),
+            Delete(target) => write!(f, "cannot delete {}", target),
+            SyntaxError(err) => write!(f, "{}", err.as_str()),
+            MultipleStarArgs => {
                 write!(f, "two starred expressions in assignment")
             }
-            CompileErrorType::InvalidStarExpr => write!(f, "cannot use starred expression here"),
-            CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"),
-            CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"),
-            CompileErrorType::InvalidReturn => write!(f, "'return' outside function"),
-            CompileErrorType::InvalidYield => write!(f, "'yield' outside function"),
-            CompileErrorType::InvalidYieldFrom => write!(f, "'yield from' outside function"),
-            CompileErrorType::InvalidAwait => write!(f, "'await' outside async function"),
-            CompileErrorType::AsyncYieldFrom => write!(f, "'yield from' inside async function"),
-            CompileErrorType::AsyncReturnValue => {
+            InvalidStarExpr => write!(f, "cannot use starred expression here"),
+            InvalidBreak => write!(f, "'break' outside loop"),
+            InvalidContinue => write!(f, "'continue' outside loop"),
+            InvalidReturn => write!(f, "'return' outside function"),
+            InvalidYield => write!(f, "'yield' outside function"),
+            InvalidYieldFrom => write!(f, "'yield from' outside function"),
+            InvalidAwait => write!(f, "'await' outside async function"),
+            AsyncYieldFrom => write!(f, "'yield from' inside async function"),
+            AsyncReturnValue => {
                 write!(f, "'return' with value inside async generator")
             }
-            CompileErrorType::InvalidFuturePlacement => write!(
+            InvalidFuturePlacement => write!(
                 f,
                 "from __future__ imports must occur at the beginning of the file"
             ),
-            CompileErrorType::InvalidFutureFeature(feat) => {
+            InvalidFutureFeature(feat) => {
                 write!(f, "future feature {} is not defined", feat)
             }
-            CompileErrorType::FunctionImportStar => {
+            FunctionImportStar => {
                 write!(f, "import * only allowed at module level")
             }
-            CompileErrorType::TooManyStarUnpack => {
+            TooManyStarUnpack => {
                 write!(f, "too many expressions in star-unpacking assignment")
             }
-            CompileErrorType::EmptyWithItems => {
+            EmptyWithItems => {
                 write!(f, "empty items on With")
             }
-            CompileErrorType::EmptyWithBody => {
+            EmptyWithBody => {
                 write!(f, "empty body on With")
             }
-            CompileErrorType::NotImplementedYet => {
+            NotImplementedYet => {
                 write!(f, "RustPython does not implement this feature yet")
             }
         }
     }
 }
 
-impl Error for CompileErrorType {}
+impl Error for CodegenErrorType {}
 
-impl fmt::Display for CompileError {
+impl fmt::Display for CodegenError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "{} at {}", self.error, self.location)
     }
 }
 
-impl Error for CompileError {
+impl Error for CodegenError {
     fn source(&self) -> Option<&(dyn Error + 'static)> {
         None
     }
diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 5c9235d8..6c2bcf51 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -8,7 +8,7 @@ Inspirational file: https://github.com/python/cpython/blob/main/Python/symtable.
 */
 
 use crate::{
-    error::{CompileError, CompileErrorType},
+    error::{CodegenError, CodegenErrorType},
     IndexMap,
 };
 use rustpython_ast::{self as ast, Location};
@@ -168,9 +168,9 @@ pub struct SymbolTableError {
 }
 
 impl SymbolTableError {
-    pub fn into_compile_error(self, source_path: String) -> CompileError {
-        CompileError {
-            error: CompileErrorType::SyntaxError(self.error),
+    pub fn into_codegen_error(self, source_path: String) -> CodegenError {
+        CodegenError {
+            error: CodegenErrorType::SyntaxError(self.error),
             location: self.location,
             source_path,
         }
diff --git a/src/lib.rs b/src/lib.rs
index f09af260..9e0a5c2e 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -10,19 +10,19 @@ pub use rustpython_bytecode::Mode;
 pub use rustpython_codegen::compile::CompileOpts;
 
 #[derive(Debug, thiserror::Error)]
-pub enum CompileErrorType {
+pub enum CodegenErrorType {
     #[error(transparent)]
-    Compile(#[from] rustpython_codegen::error::CompileErrorType),
+    Compile(#[from] rustpython_codegen::error::CodegenErrorType),
     #[error(transparent)]
     Parse(#[from] rustpython_parser::error::ParseErrorType),
 }
 
 #[derive(Debug, thiserror::Error)]
 pub struct CompileError {
-    pub error: CompileErrorType,
-    pub statement: Option<String>,
+    pub error: CodegenErrorType,
     pub source_path: String,
     pub location: Location,
+    pub statement: Option<String>,
 }
 
 impl fmt::Display for CompileError {
@@ -42,7 +42,7 @@ impl fmt::Display for CompileError {
 }
 
 impl CompileError {
-    fn from_compile(error: rustpython_codegen::error::CompileError, source: &str) -> Self {
+    fn from_codegen(error: rustpython_codegen::error::CodegenError, source: &str) -> Self {
         CompileError {
             error: error.error.into(),
             location: error.location,
@@ -67,7 +67,7 @@ impl CompileError {
         source: &str,
         source_path: String,
     ) -> Self {
-        Self::from_compile(error.into_compile_error(source_path), source)
+        Self::from_codegen(error.into_codegen_error(source_path), source)
     }
 }
 
@@ -93,7 +93,7 @@ pub fn compile(
             .unwrap_or_else(|e| match e {});
     }
     compile::compile_top(&ast, source_path, mode, opts)
-        .map_err(|e| CompileError::from_compile(e, source))
+        .map_err(|e| CompileError::from_codegen(e, source))
 }
 
 pub fn compile_symtable(

From 03ccd4b9d89a642fcf055dcb8f13f8dd371f9968 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 06:48:47 +0900
Subject: [PATCH 256/459] Add source_path to ParseError

---
 codegen/src/compile.rs |  2 +-
 parser/src/error.rs    | 14 +++++++++++--
 parser/src/fstring.rs  |  2 +-
 parser/src/lib.rs      |  2 +-
 parser/src/parser.rs   | 46 +++++++++++++++++++++---------------------
 parser/src/string.rs   | 16 +++++++--------
 src/lib.rs             | 22 +++++++++-----------
 7 files changed, 55 insertions(+), 49 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index f7dd2727..59d9faaf 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -2707,7 +2707,7 @@ mod tests {
             "source_path".to_owned(),
             "<module>".to_owned(),
         );
-        let ast = parser::parse_program(source).unwrap();
+        let ast = parser::parse_program(source, "<test>").unwrap();
         let symbol_scope = SymbolTable::scan_program(&ast).unwrap();
         compiler.compile_program(&ast, symbol_scope).unwrap();
         compiler.pop_code_object()
diff --git a/parser/src/error.rs b/parser/src/error.rs
index cc53ef2f..d256e2a7 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -124,6 +124,7 @@ impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
 pub struct ParseError {
     pub error: ParseErrorType,
     pub location: Location,
+    pub source_path: String,
 }
 
 #[derive(Debug, PartialEq)]
@@ -141,21 +142,28 @@ pub enum ParseErrorType {
 }
 
 /// Convert `lalrpop_util::ParseError` to our internal type
-impl From<LalrpopError<Location, Tok, LexicalError>> for ParseError {
-    fn from(err: LalrpopError<Location, Tok, LexicalError>) -> Self {
+impl ParseError {
+    pub(crate) fn from_lalrpop(
+        err: LalrpopError<Location, Tok, LexicalError>,
+        source_path: &str,
+    ) -> Self {
+        let source_path = source_path.to_owned();
         match err {
             // TODO: Are there cases where this isn't an EOF?
             LalrpopError::InvalidToken { location } => ParseError {
                 error: ParseErrorType::Eof,
                 location,
+                source_path,
             },
             LalrpopError::ExtraToken { token } => ParseError {
                 error: ParseErrorType::ExtraToken(token.1),
                 location: token.0,
+                source_path,
             },
             LalrpopError::User { error } => ParseError {
                 error: ParseErrorType::Lexical(error.error),
                 location: error.location,
+                source_path,
             },
             LalrpopError::UnrecognizedToken { token, expected } => {
                 // Hacky, but it's how CPython does it. See PyParser_AddToken,
@@ -164,11 +172,13 @@ impl From<LalrpopError<Location, Tok, LexicalError>> for ParseError {
                 ParseError {
                     error: ParseErrorType::UnrecognizedToken(token.1, expected),
                     location: token.0,
+                    source_path,
                 }
             }
             LalrpopError::UnrecognizedEOF { location, .. } => ParseError {
                 error: ParseErrorType::Eof,
                 location,
+                source_path,
             },
         }
     }
diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index 28693061..11d75a09 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -293,7 +293,7 @@ impl<'a> FStringParser<'a> {
 
 fn parse_fstring_expr(source: &str) -> Result<Expr, ParseError> {
     let fstring_body = format!("({})", source);
-    parse_expression(&fstring_body)
+    parse_expression(&fstring_body, "<fstring>")
 }
 
 /// Parse an fstring from a string, located at a certain position in the sourcecode.
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index 694e0a50..f65f7369 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -11,7 +11,7 @@
 //! use rustpython_parser::{parser, ast};
 //!
 //! let python_source = "print('Hello world')";
-//! let python_ast = parser::parse_expression(python_source).unwrap();
+//! let python_ast = parser::parse_expression(python_source, "<embedded>").unwrap();
 //!
 //! ```
 
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 990f846f..3199701d 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -20,8 +20,8 @@ use crate::python;
  */
 
 /// Parse a full python program, containing usually multiple lines.
-pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
-    parse(source, Mode::Module).map(|top| match top {
+pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, ParseError> {
+    parse(source, Mode::Module, source_path).map(|top| match top {
         ast::Mod::Module { body, .. } => body,
         _ => unreachable!(),
     })
@@ -33,7 +33,7 @@ pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
 /// ```
 /// extern crate num_bigint;
 /// use rustpython_parser::{parser, ast};
-/// let expr = parser::parse_expression("1 + 2").unwrap();
+/// let expr = parser::parse_expression("1 + 2", "<embedded>").unwrap();
 ///
 /// assert_eq!(
 ///     expr,
@@ -63,22 +63,22 @@ pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
 /// );
 ///
 /// ```
-pub fn parse_expression(source: &str) -> Result<ast::Expr, ParseError> {
-    parse(source, Mode::Expression).map(|top| match top {
+pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> {
+    parse(source, Mode::Expression, path).map(|top| match top {
         ast::Mod::Expression { body } => *body,
         _ => unreachable!(),
     })
 }
 
 // Parse a given source code
-pub fn parse(source: &str, mode: Mode) -> Result<ast::Mod, ParseError> {
+pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, ParseError> {
     let lxr = lexer::make_tokenizer(source);
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
     let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
 
     python::TopParser::new()
         .parse(tokenizer)
-        .map_err(ParseError::from)
+        .map_err(|e| ParseError::from_lalrpop(e, source_path))
 }
 
 #[cfg(test)]
@@ -87,56 +87,56 @@ mod tests {
 
     #[test]
     fn test_parse_empty() {
-        let parse_ast = parse_program("").unwrap();
+        let parse_ast = parse_program("", "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_string() {
         let source = String::from("'Hello world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string() {
         let source = String::from("f'Hello world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_print_hello() {
         let source = String::from("print('Hello world')");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_print_2() {
         let source = String::from("print('Hello world', 2)");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_kwargs() {
         let source = String::from("my_func('positional', keyword=2)");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_if_elif_else() {
         let source = String::from("if 1: 10\nelif 2: 20\nelse: 30");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_lambda() {
         let source = "lambda x, y: x * y"; // lambda(x, y): x * y";
-        let parse_ast = parse_program(source).unwrap();
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
@@ -144,7 +144,7 @@ mod tests {
     fn test_parse_tuples() {
         let source = "a, b = 4, 5";
 
-        insta::assert_debug_snapshot!(parse_program(source).unwrap());
+        insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
     }
 
     #[test]
@@ -155,48 +155,48 @@ class Foo(A, B):
   pass
  def method_with_default(self, arg='default'):
   pass";
-        insta::assert_debug_snapshot!(parse_program(source).unwrap());
+        insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
     }
 
     #[test]
     fn test_parse_dict_comprehension() {
         let source = String::from("{x1: x2 for y in z}");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_list_comprehension() {
         let source = String::from("[x for y in z]");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_double_list_comprehension() {
         let source = String::from("[x for y, y2 in z for a in b if a < 5 if a > 10]");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_generator_comprehension() {
         let source = String::from("(x for y in z)");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_named_expression_generator_comprehension() {
         let source = String::from("(x := y + 1 for y in z)");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_if_else_generator_comprehension() {
         let source = String::from("(x if y else y for y in z)");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 }
diff --git a/parser/src/string.rs b/parser/src/string.rs
index 02a55145..1c16f7f6 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -88,56 +88,56 @@ mod tests {
     #[test]
     fn test_parse_string_concat() {
         let source = String::from("'Hello ' 'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_string_concat_1() {
         let source = String::from("'Hello ' u'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_string_concat_2() {
         let source = String::from("u'Hello ' 'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string_concat_1() {
         let source = String::from("'Hello ' f'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string_concat_2() {
         let source = String::from("'Hello ' f'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string_concat_3() {
         let source = String::from("'Hello ' f'world{\"!\"}'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_f_string_concat_1() {
         let source = String::from("u'Hello ' f'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_f_string_concat_2() {
         let source = String::from("u'Hello ' f'world' '!'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 }
diff --git a/src/lib.rs b/src/lib.rs
index 9e0a5c2e..6996589a 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -43,22 +43,18 @@ impl fmt::Display for CompileError {
 
 impl CompileError {
     fn from_codegen(error: rustpython_codegen::error::CodegenError, source: &str) -> Self {
-        CompileError {
+        Self {
             error: error.error.into(),
             location: error.location,
             source_path: error.source_path,
             statement: get_statement(source, error.location),
         }
     }
-    fn from_parse(
-        error: rustpython_parser::error::ParseError,
-        source: &str,
-        source_path: String,
-    ) -> Self {
-        CompileError {
+    fn from_parse(error: rustpython_parser::error::ParseError, source: &str) -> Self {
+        Self {
             error: error.error.into(),
             location: error.location,
-            source_path,
+            source_path: error.source_path,
             statement: get_statement(source, error.location),
         }
     }
@@ -83,9 +79,9 @@ pub fn compile(
         compile::Mode::Eval => parser::Mode::Expression,
         compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive,
     };
-    let mut ast = match parser::parse(source, parser_mode) {
+    let mut ast = match parser::parse(source, parser_mode, &source_path) {
         Ok(x) => x,
-        Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
+        Err(e) => return Err(CompileError::from_parse(e, source)),
     };
     if opts.optimize > 0 {
         ast = ConstantOptimizer::new()
@@ -101,14 +97,14 @@ pub fn compile_symtable(
     mode: compile::Mode,
     source_path: &str,
 ) -> Result<symboltable::SymbolTable, CompileError> {
-    let parse_err = |e| CompileError::from_parse(e, source, source_path.to_owned());
+    let parse_err = |e| CompileError::from_parse(e, source);
     let res = match mode {
         compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
-            let ast = parser::parse_program(source).map_err(parse_err)?;
+            let ast = parser::parse_program(source, source_path).map_err(parse_err)?;
             symboltable::SymbolTable::scan_program(&ast)
         }
         compile::Mode::Eval => {
-            let expr = parser::parse_expression(source).map_err(parse_err)?;
+            let expr = parser::parse_expression(source, source_path).map_err(parse_err)?;
             symboltable::SymbolTable::scan_expr(&expr)
         }
     };

From 0e0433b87b67213090def3246b69464ef83602fa Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 06:58:28 +0900
Subject: [PATCH 257/459] clean up cargo.toml sort

---
 Cargo.toml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index df762199..c2e22b15 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,7 +6,8 @@ authors = ["RustPython Team"]
 edition = "2021"
 
 [dependencies]
-thiserror = "1.0"
 rustpython-codegen = { path = "codegen" }
 rustpython-parser = { path = "parser" }
 rustpython-bytecode = { path = "bytecode" }
+
+thiserror = "1.0"

From d60ad9be23654fb5a149004aeb18fdcc3d68d849 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 07:12:10 +0900
Subject: [PATCH 258/459] sort Cargo.toml dependencies

---
 Cargo.toml          |  2 +-
 ast/Cargo.toml      |  2 +-
 bytecode/Cargo.toml |  5 ++---
 codegen/Cargo.toml  | 10 ++++++----
 parser/Cargo.toml   |  5 +++--
 5 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index c2e22b15..00fc9147 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,8 +6,8 @@ authors = ["RustPython Team"]
 edition = "2021"
 
 [dependencies]
+rustpython-bytecode = { path = "bytecode" }
 rustpython-codegen = { path = "codegen" }
 rustpython-parser = { path = "parser" }
-rustpython-bytecode = { path = "bytecode" }
 
 thiserror = "1.0"
diff --git a/ast/Cargo.toml b/ast/Cargo.toml
index 144ba621..39eedf4f 100644
--- a/ast/Cargo.toml
+++ b/ast/Cargo.toml
@@ -12,5 +12,5 @@ unparse = ["rustpython-common"]
 
 [dependencies]
 num-bigint = "0.4.3"
+rustpython-bytecode = { path = "../bytecode" }
 rustpython-common = { path = "../../common", optional = true }
-rustpython-bytecode = { path = "../bytecode"}
diff --git a/bytecode/Cargo.toml b/bytecode/Cargo.toml
index 1457072b..05d698ed 100644
--- a/bytecode/Cargo.toml
+++ b/bytecode/Cargo.toml
@@ -7,14 +7,13 @@ edition = "2021"
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
-
 [dependencies]
 bincode = "1.3.3"
 bitflags = "1.3.2"
+bstr = "0.2.17"
+itertools = "0.10.3"
 lz4_flex = "0.9.2"
 num-bigint = { version = "0.4.3", features = ["serde"] }
 num-complex = { version = "0.4.0", features = ["serde"] }
 serde = { version = "1.0.136", features = ["derive"] }
-itertools = "0.10.3"
-bstr = "0.2.17"
 static_assertions = "1.1.0"
diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml
index 3a770df3..77cd7073 100644
--- a/codegen/Cargo.toml
+++ b/codegen/Cargo.toml
@@ -8,15 +8,17 @@ license = "MIT"
 edition = "2021"
 
 [dependencies]
+rustpython-ast = { path = "../ast", features = ["unparse"] }
+rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
+
+ahash = "0.7.6"
 indexmap = "1.8.1"
 itertools = "0.10.3"
-rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
-rustpython-ast = { path = "../ast", features = ["unparse"] }
+log = "0.4.16"
 num-complex = { version = "0.4.0", features = ["serde"] }
 num-traits = "0.2.14"
-log = "0.4.16"
-ahash = "0.7.6"
 
 [dev-dependencies]
 rustpython-parser = { path = "../parser" }
+
 insta = "1.14.0"
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index 60dabdcc..2522e726 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -2,7 +2,7 @@
 name = "rustpython-parser"
 version = "0.1.2"
 description = "Parser for python code."
-authors = [ "RustPython Team" ]
+authors = ["RustPython Team"]
 build = "build.rs"
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
@@ -15,6 +15,8 @@ phf_codegen = "0.10"
 tiny-keccak = { version = "2", features = ["sha3"] }
 
 [dependencies]
+rustpython-ast = { path = "../ast" }
+
 ahash = "0.7.6"
 itertools = "0.10.3"
 lalrpop-util = "0.19.8"
@@ -22,7 +24,6 @@ log = "0.4.16"
 num-bigint = "0.4.3"
 num-traits = "0.2.14"
 phf = "0.10.1"
-rustpython-ast = { path = "../ast" }
 unic-emoji-char = "0.9.0"
 unic-ucd-ident  = "0.9.0"
 unicode_names2 = "0.5.0"

From b3095c74516a5ef9ca00d99b67f7848cdc325547 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 07:34:30 +0900
Subject: [PATCH 259/459] rustpython-bytecode -> rustpython-compiler-core

---
 Cargo.toml                                  |  2 +-
 ast/Cargo.toml                              |  2 +-
 ast/src/constant.rs                         |  2 +-
 codegen/Cargo.toml                          |  2 +-
 codegen/src/compile.rs                      |  6 +++---
 codegen/src/ir.rs                           |  2 +-
 {bytecode => core}/Cargo.toml               |  2 +-
 bytecode/src/lib.rs => core/src/bytecode.rs | 18 ++++++------------
 core/src/lib.rs                             |  8 ++++++++
 {bytecode => core}/src/mode.rs              |  0
 src/lib.rs                                  |  4 ++--
 11 files changed, 25 insertions(+), 23 deletions(-)
 rename {bytecode => core}/Cargo.toml (93%)
 rename bytecode/src/lib.rs => core/src/bytecode.rs (98%)
 create mode 100644 core/src/lib.rs
 rename {bytecode => core}/src/mode.rs (100%)

diff --git a/Cargo.toml b/Cargo.toml
index 00fc9147..118286f4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,7 +6,7 @@ authors = ["RustPython Team"]
 edition = "2021"
 
 [dependencies]
-rustpython-bytecode = { path = "bytecode" }
+rustpython-compiler-core = { path = "core" }
 rustpython-codegen = { path = "codegen" }
 rustpython-parser = { path = "parser" }
 
diff --git a/ast/Cargo.toml b/ast/Cargo.toml
index 39eedf4f..7efefb3a 100644
--- a/ast/Cargo.toml
+++ b/ast/Cargo.toml
@@ -12,5 +12,5 @@ unparse = ["rustpython-common"]
 
 [dependencies]
 num-bigint = "0.4.3"
-rustpython-bytecode = { path = "../bytecode" }
+rustpython-compiler-core = { path = "../core" }
 rustpython-common = { path = "../../common", optional = true }
diff --git a/ast/src/constant.rs b/ast/src/constant.rs
index 1d165594..b73bab62 100644
--- a/ast/src/constant.rs
+++ b/ast/src/constant.rs
@@ -1,5 +1,5 @@
 use num_bigint::BigInt;
-pub use rustpython_bytecode::ConversionFlag;
+pub use rustpython_compiler_core::ConversionFlag;
 
 #[derive(Debug, PartialEq)]
 pub enum Constant {
diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml
index 77cd7073..424d85a6 100644
--- a/codegen/Cargo.toml
+++ b/codegen/Cargo.toml
@@ -9,7 +9,7 @@ edition = "2021"
 
 [dependencies]
 rustpython-ast = { path = "../ast", features = ["unparse"] }
-rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
+rustpython-compiler-core = { path = "../core", version = "0.1.1" }
 
 ahash = "0.7.6"
 indexmap = "1.8.1"
diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 59d9faaf..0b593588 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -15,10 +15,10 @@ use itertools::Itertools;
 use num_complex::Complex64;
 use num_traits::ToPrimitive;
 use rustpython_ast as ast;
-use rustpython_bytecode::{self as bytecode, CodeObject, ConstantData, Instruction};
+use rustpython_compiler_core::{self as bytecode, CodeObject, ConstantData, Instruction};
 use std::borrow::Cow;
 
-pub use rustpython_bytecode::Mode;
+pub use rustpython_compiler_core::Mode;
 
 type CompileResult<T> = Result<T, CodegenError>;
 
@@ -2698,7 +2698,7 @@ fn compile_constant(value: &ast::Constant) -> ConstantData {
 mod tests {
     use super::{CompileOpts, Compiler};
     use crate::symboltable::SymbolTable;
-    use rustpython_bytecode::CodeObject;
+    use rustpython_compiler_core::CodeObject;
     use rustpython_parser::parser;
 
     fn compile_exec(source: &str) -> CodeObject {
diff --git a/codegen/src/ir.rs b/codegen/src/ir.rs
index 39c3ad1f..c1ed1dd7 100644
--- a/codegen/src/ir.rs
+++ b/codegen/src/ir.rs
@@ -1,5 +1,5 @@
 use crate::IndexSet;
-use rustpython_bytecode::{CodeFlags, CodeObject, ConstantData, Instruction, Label, Location};
+use rustpython_compiler_core::{CodeFlags, CodeObject, ConstantData, Instruction, Label, Location};
 
 pub type BlockIdx = Label;
 
diff --git a/bytecode/Cargo.toml b/core/Cargo.toml
similarity index 93%
rename from bytecode/Cargo.toml
rename to core/Cargo.toml
index 05d698ed..5821ddc3 100644
--- a/bytecode/Cargo.toml
+++ b/core/Cargo.toml
@@ -1,5 +1,5 @@
 [package]
-name = "rustpython-bytecode"
+name = "rustpython-compiler-core"
 description = "RustPython specific bytecode."
 version = "0.1.2"
 authors = ["RustPython Team"]
diff --git a/bytecode/src/lib.rs b/core/src/bytecode.rs
similarity index 98%
rename from bytecode/src/lib.rs
rename to core/src/bytecode.rs
index c53697bc..8e9cc75b 100644
--- a/bytecode/src/lib.rs
+++ b/core/src/bytecode.rs
@@ -1,12 +1,6 @@
 //! Implement python as a virtual machine with bytecodes. This module
 //! implements bytecode structure.
 
-#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
-#![doc(html_root_url = "https://docs.rs/rustpython-bytecode/")]
-
-mod mode;
-pub use mode::Mode;
-
 use bitflags::bitflags;
 use bstr::ByteSlice;
 use itertools::Itertools;
@@ -27,7 +21,7 @@ impl Location {
     ///
     /// # Example
     /// ```
-    /// use rustpython_bytecode::Location;
+    /// use rustpython_compiler_core::Location;
     /// let loc = Location::new(10, 10);
     /// ```
     pub fn new(row: usize, column: usize) -> Self {
@@ -433,7 +427,7 @@ bitflags! {
 ///
 /// # Examples
 /// ```
-/// use rustpython_bytecode::ConstantData;
+/// use rustpython_compiler_core::ConstantData;
 /// let a = ConstantData::Float {value: 120f64};
 /// let b = ConstantData::Boolean {value: false};
 /// assert_ne!(a, b);
@@ -599,8 +593,8 @@ pub enum TestOperator {
 /// # Examples
 ///
 /// ```
-/// use rustpython_bytecode::Instruction::BinaryOperation;
-/// use rustpython_bytecode::BinaryOperator::Add;
+/// use rustpython_compiler_core::Instruction::BinaryOperation;
+/// use rustpython_compiler_core::BinaryOperator::Add;
 /// let op = BinaryOperation {op: Add};
 /// ```
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -943,7 +937,7 @@ impl Instruction {
     /// # Examples
     ///
     /// ```
-    /// use rustpython_bytecode::{Instruction, Label};
+    /// use rustpython_compiler_core::{Instruction, Label};
     /// let label = Label(0xF);
     /// let jump_inst = Instruction::Jump {target: label};
     /// assert!(jump_inst.unconditional_branch())
@@ -960,7 +954,7 @@ impl Instruction {
     /// # Examples
     ///
     /// ```
-    /// use rustpython_bytecode::{Instruction, Label, UnaryOperator};
+    /// use rustpython_compiler_core::{Instruction, Label, UnaryOperator};
     /// let jump_instruction = Instruction::Jump {target: Label(0xF)};
     /// let invert_instruction = Instruction::UnaryOperation {op: UnaryOperator::Invert};
     /// assert_eq!(jump_instruction.stack_effect(true), 0);
diff --git a/core/src/lib.rs b/core/src/lib.rs
new file mode 100644
index 00000000..ac798da5
--- /dev/null
+++ b/core/src/lib.rs
@@ -0,0 +1,8 @@
+#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
+#![doc(html_root_url = "https://docs.rs/rustpython-compiler-core/")]
+
+pub mod bytecode;
+mod mode;
+
+pub use bytecode::*;
+pub use mode::Mode;
diff --git a/bytecode/src/mode.rs b/core/src/mode.rs
similarity index 100%
rename from bytecode/src/mode.rs
rename to core/src/mode.rs
diff --git a/src/lib.rs b/src/lib.rs
index 6996589a..5f3e72cc 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,13 +1,13 @@
-use rustpython_bytecode::CodeObject;
 use rustpython_codegen::{compile, symboltable};
+use rustpython_compiler_core::CodeObject;
 use rustpython_parser::{
     ast::{fold::Fold, ConstantOptimizer, Location},
     parser,
 };
 use std::fmt;
 
-pub use rustpython_bytecode::Mode;
 pub use rustpython_codegen::compile::CompileOpts;
+pub use rustpython_compiler_core::Mode;
 
 #[derive(Debug, thiserror::Error)]
 pub enum CodegenErrorType {

From a22abc555077d70605deaca1cd0ed67c85601324 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 07:40:15 +0900
Subject: [PATCH 260/459] Share location between compiler crates

---
 ast/src/location.rs  | 48 ++++++++++++++---------------------------
 core/src/bytecode.rs | 33 +---------------------------
 core/src/lib.rs      |  4 +++-
 core/src/location.rs | 51 ++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 71 insertions(+), 65 deletions(-)
 create mode 100644 core/src/location.rs

diff --git a/ast/src/location.rs b/ast/src/location.rs
index 830cb5fe..95843202 100644
--- a/ast/src/location.rs
+++ b/ast/src/location.rs
@@ -4,14 +4,24 @@ use std::fmt;
 
 /// A location somewhere in the sourcecode.
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
-pub struct Location {
-    row: usize,
-    column: usize,
+pub struct Location(rustpython_compiler_core::Location);
+
+impl std::ops::Deref for Location {
+    type Target = rustpython_compiler_core::Location;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl std::ops::DerefMut for Location {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
 }
 
 impl fmt::Display for Location {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "line {} column {}", self.row, self.column)
+        write!(f, "line {} column {}", self.row(), self.column())
     }
 }
 
@@ -33,7 +43,7 @@ impl Location {
                     "{}\n{}{arrow:>pad$}",
                     self.desc,
                     self.line,
-                    pad = self.loc.column,
+                    pad = self.loc.column(),
                     arrow = "^",
                 )
             }
@@ -48,32 +58,6 @@ impl Location {
 
 impl Location {
     pub fn new(row: usize, column: usize) -> Self {
-        Location { row, column }
-    }
-
-    pub fn row(&self) -> usize {
-        self.row
-    }
-
-    pub fn column(&self) -> usize {
-        self.column
-    }
-
-    pub fn reset(&mut self) {
-        self.row = 1;
-        self.column = 1;
-    }
-
-    pub fn go_right(&mut self) {
-        self.column += 1;
-    }
-
-    pub fn go_left(&mut self) {
-        self.column -= 1;
-    }
-
-    pub fn newline(&mut self) {
-        self.row += 1;
-        self.column = 1;
+        Location(rustpython_compiler_core::Location::new(row, column))
     }
 }
diff --git a/core/src/bytecode.rs b/core/src/bytecode.rs
index 8e9cc75b..ae28f6ef 100644
--- a/core/src/bytecode.rs
+++ b/core/src/bytecode.rs
@@ -1,6 +1,7 @@
 //! Implement python as a virtual machine with bytecodes. This module
 //! implements bytecode structure.
 
+use crate::Location;
 use bitflags::bitflags;
 use bstr::ByteSlice;
 use itertools::Itertools;
@@ -9,38 +10,6 @@ use num_complex::Complex64;
 use serde::{Deserialize, Serialize};
 use std::{collections::BTreeSet, fmt, hash};
 
-/// Sourcecode location.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Location {
-    row: u32,
-    column: u32,
-}
-
-impl Location {
-    /// Creates a new Location object at the given row and column.
-    ///
-    /// # Example
-    /// ```
-    /// use rustpython_compiler_core::Location;
-    /// let loc = Location::new(10, 10);
-    /// ```
-    pub fn new(row: usize, column: usize) -> Self {
-        let row = row.try_into().expect("Location::row over u32");
-        let column = column.try_into().expect("Location::column over u32");
-        Location { row, column }
-    }
-
-    /// Current row
-    pub fn row(&self) -> usize {
-        self.row as usize
-    }
-
-    /// Current column
-    pub fn column(&self) -> usize {
-        self.column as usize
-    }
-}
-
 pub trait Constant: Sized {
     type Name: AsRef<str>;
 
diff --git a/core/src/lib.rs b/core/src/lib.rs
index ac798da5..0b2e23e8 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -1,8 +1,10 @@
 #![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
 #![doc(html_root_url = "https://docs.rs/rustpython-compiler-core/")]
 
-pub mod bytecode;
+mod bytecode;
+mod location;
 mod mode;
 
 pub use bytecode::*;
+pub use location::Location;
 pub use mode::Mode;
diff --git a/core/src/location.rs b/core/src/location.rs
new file mode 100644
index 00000000..1763763b
--- /dev/null
+++ b/core/src/location.rs
@@ -0,0 +1,51 @@
+use serde::{Deserialize, Serialize};
+
+/// Sourcecode location.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Location {
+    pub(super) row: u32,
+    pub(super) column: u32,
+}
+
+impl Location {
+    /// Creates a new Location object at the given row and column.
+    ///
+    /// # Example
+    /// ```
+    /// use rustpython_compiler_core::Location;
+    /// let loc = Location::new(10, 10);
+    /// ```
+    pub fn new(row: usize, column: usize) -> Self {
+        let row = row.try_into().expect("Location::row over u32");
+        let column = column.try_into().expect("Location::column over u32");
+        Location { row, column }
+    }
+
+    /// Current row
+    pub fn row(&self) -> usize {
+        self.row as usize
+    }
+
+    /// Current column
+    pub fn column(&self) -> usize {
+        self.column as usize
+    }
+
+    pub fn reset(&mut self) {
+        self.row = 1;
+        self.column = 1;
+    }
+
+    pub fn go_right(&mut self) {
+        self.column += 1;
+    }
+
+    pub fn go_left(&mut self) {
+        self.column -= 1;
+    }
+
+    pub fn newline(&mut self) {
+        self.row += 1;
+        self.column = 1;
+    }
+}

From 9d6ab72d799bdd1354b2298857e4fe2f21aff479 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 08:38:21 +0900
Subject: [PATCH 261/459] integrate ast::Location into compilre-core::Location

---
 ast/asdl_rs.py             |  2 +-
 ast/src/ast_gen.rs         |  2 +-
 ast/src/lib.rs             |  3 +-
 ast/src/location.rs        | 63 --------------------------------------
 codegen/src/compile.rs     | 12 ++++----
 codegen/src/error.rs       |  5 ++-
 codegen/src/symboltable.rs |  3 +-
 core/src/location.rs       | 10 ++++++
 parser/src/error.rs        | 11 +++----
 parser/src/function.rs     |  5 ++-
 parser/src/parser.rs       |  8 ++---
 src/lib.rs                 |  9 ++----
 12 files changed, 34 insertions(+), 99 deletions(-)
 delete mode 100644 ast/src/location.rs

diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py
index a4c894aa..71c90328 100755
--- a/ast/asdl_rs.py
+++ b/ast/asdl_rs.py
@@ -645,7 +645,7 @@ def write_ast_def(mod, typeinfo, f):
         #![allow(clippy::derive_partial_eq_without_eq)]
         
         pub use crate::constant::*;
-        pub use crate::location::Location;
+        pub use crate::Location;
 
         type Ident = String;
         \n
diff --git a/ast/src/ast_gen.rs b/ast/src/ast_gen.rs
index fda39b84..17900fe3 100644
--- a/ast/src/ast_gen.rs
+++ b/ast/src/ast_gen.rs
@@ -3,7 +3,7 @@
 #![allow(clippy::derive_partial_eq_without_eq)]
 
 pub use crate::constant::*;
-pub use crate::location::Location;
+pub use crate::Location;
 
 type Ident = String;
 
diff --git a/ast/src/lib.rs b/ast/src/lib.rs
index b6969986..d668bede 100644
--- a/ast/src/lib.rs
+++ b/ast/src/lib.rs
@@ -3,11 +3,10 @@ mod constant;
 #[cfg(feature = "fold")]
 mod fold_helpers;
 mod impls;
-mod location;
 #[cfg(feature = "unparse")]
 mod unparse;
 
 pub use ast_gen::*;
-pub use location::Location;
+pub use rustpython_compiler_core::Location;
 
 pub type Suite<U = ()> = Vec<Stmt<U>>;
diff --git a/ast/src/location.rs b/ast/src/location.rs
deleted file mode 100644
index 95843202..00000000
--- a/ast/src/location.rs
+++ /dev/null
@@ -1,63 +0,0 @@
-//! Datatypes to support source location information.
-
-use std::fmt;
-
-/// A location somewhere in the sourcecode.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
-pub struct Location(rustpython_compiler_core::Location);
-
-impl std::ops::Deref for Location {
-    type Target = rustpython_compiler_core::Location;
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl std::ops::DerefMut for Location {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
-}
-
-impl fmt::Display for Location {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "line {} column {}", self.row(), self.column())
-    }
-}
-
-impl Location {
-    pub fn visualize<'a>(
-        &self,
-        line: &'a str,
-        desc: impl fmt::Display + 'a,
-    ) -> impl fmt::Display + 'a {
-        struct Visualize<'a, D: fmt::Display> {
-            loc: Location,
-            line: &'a str,
-            desc: D,
-        }
-        impl<D: fmt::Display> fmt::Display for Visualize<'_, D> {
-            fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-                write!(
-                    f,
-                    "{}\n{}{arrow:>pad$}",
-                    self.desc,
-                    self.line,
-                    pad = self.loc.column(),
-                    arrow = "^",
-                )
-            }
-        }
-        Visualize {
-            loc: *self,
-            line,
-            desc,
-        }
-    }
-}
-
-impl Location {
-    pub fn new(row: usize, column: usize) -> Self {
-        Location(rustpython_compiler_core::Location::new(row, column))
-    }
-}
diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 0b593588..6cf33acc 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -15,7 +15,7 @@ use itertools::Itertools;
 use num_complex::Complex64;
 use num_traits::ToPrimitive;
 use rustpython_ast as ast;
-use rustpython_compiler_core::{self as bytecode, CodeObject, ConstantData, Instruction};
+use rustpython_compiler_core::{self as bytecode, CodeObject, ConstantData, Instruction, Location};
 use std::borrow::Cow;
 
 pub use rustpython_compiler_core::Mode;
@@ -64,7 +64,7 @@ struct Compiler {
     code_stack: Vec<ir::CodeInfo>,
     symbol_table_stack: Vec<SymbolTable>,
     source_path: String,
-    current_source_location: ast::Location,
+    current_source_location: Location,
     qualified_path: Vec<String>,
     done_with_future_stmts: bool,
     future_annotations: bool,
@@ -220,7 +220,7 @@ impl Compiler {
             code_stack: vec![module_code],
             symbol_table_stack: Vec::new(),
             source_path,
-            current_source_location: ast::Location::default(),
+            current_source_location: Location::default(),
             qualified_path: Vec::new(),
             done_with_future_stmts: false,
             future_annotations: false,
@@ -237,7 +237,7 @@ impl Compiler {
     fn error(&self, error: CodegenErrorType) -> CodegenError {
         self.error_loc(error, self.current_source_location)
     }
-    fn error_loc(&self, error: CodegenErrorType, location: ast::Location) -> CodegenError {
+    fn error_loc(&self, error: CodegenErrorType, location: Location) -> CodegenError {
         CodegenError {
             error,
             location,
@@ -2617,7 +2617,7 @@ impl Compiler {
         code.current_block = block;
     }
 
-    fn set_source_location(&mut self, location: ast::Location) {
+    fn set_source_location(&mut self, location: Location) {
         self.current_source_location = location;
     }
 
@@ -2672,7 +2672,7 @@ fn try_get_constant_string(values: &[ast::Expr]) -> Option<String> {
     }
 }
 
-fn compile_location(location: &ast::Location) -> bytecode::Location {
+fn compile_location(location: &Location) -> bytecode::Location {
     bytecode::Location::new(location.row(), location.column())
 }
 
diff --git a/codegen/src/error.rs b/codegen/src/error.rs
index 60444a20..8c3ea3f2 100644
--- a/codegen/src/error.rs
+++ b/codegen/src/error.rs
@@ -1,5 +1,4 @@
-use rustpython_ast::Location;
-
+use rustpython_compiler_core::Location;
 use std::{error::Error, fmt};
 
 #[derive(Debug)]
@@ -91,7 +90,7 @@ impl Error for CodegenErrorType {}
 
 impl fmt::Display for CodegenError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{} at {}", self.error, self.location)
+        self.location.fmt_with(f, &self.error)
     }
 }
 
diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 6c2bcf51..1044f981 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -11,7 +11,8 @@ use crate::{
     error::{CodegenError, CodegenErrorType},
     IndexMap,
 };
-use rustpython_ast::{self as ast, Location};
+use rustpython_ast as ast;
+use rustpython_compiler_core::Location;
 use std::{borrow::Cow, fmt};
 
 /// Captures all symbols in the current scope, and has a list of subscopes in this scope.
diff --git a/core/src/location.rs b/core/src/location.rs
index 1763763b..fedcdcf7 100644
--- a/core/src/location.rs
+++ b/core/src/location.rs
@@ -7,6 +7,16 @@ pub struct Location {
     pub(super) column: u32,
 }
 
+impl Location {
+    pub fn fmt_with(
+        &self,
+        f: &mut std::fmt::Formatter,
+        e: &impl std::fmt::Display,
+    ) -> std::fmt::Result {
+        write!(f, "{} at line {} column {}", e, self.row(), self.column())
+    }
+}
+
 impl Location {
     /// Creates a new Location object at the given row and column.
     ///
diff --git a/parser/src/error.rs b/parser/src/error.rs
index d256e2a7..76f02dc5 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -1,12 +1,9 @@
 //! Define internal parse error types
 //! The goal is to provide a matching and a safe error API, maksing errors from LALR
-use lalrpop_util::ParseError as LalrpopError;
-
-use crate::ast::Location;
-use crate::token::Tok;
 
-use std::error::Error;
-use std::fmt;
+use crate::{ast::Location, token::Tok};
+use lalrpop_util::ParseError as LalrpopError;
+use std::{error::Error, fmt};
 
 /// Represents an error during lexical scanning.
 #[derive(Debug, PartialEq)]
@@ -186,7 +183,7 @@ impl ParseError {
 
 impl fmt::Display for ParseError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{} at {}", self.error, self.location)
+        self.location.fmt_with(f, &self.error)
     }
 }
 
diff --git a/parser/src/function.rs b/parser/src/function.rs
index 68d890f2..2a384ddf 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -1,8 +1,7 @@
-use ahash::RandomState;
-use std::collections::HashSet;
-
 use crate::ast;
 use crate::error::{LexicalError, LexicalErrorType};
+use ahash::RandomState;
+use std::collections::HashSet;
 
 pub struct ArgumentList {
     pub args: Vec<ast::Expr>,
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 3199701d..ff86ef40 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -5,13 +5,9 @@
 //! parse a whole program, a single statement, or a single
 //! expression.
 
-use std::iter;
-
-use crate::ast;
-use crate::error::ParseError;
-use crate::lexer;
 pub use crate::mode::Mode;
-use crate::python;
+use crate::{ast, error::ParseError, lexer, python};
+use std::iter;
 
 /*
  * Parse python code.
diff --git a/src/lib.rs b/src/lib.rs
index 5f3e72cc..a96eefa4 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -30,13 +30,10 @@ impl fmt::Display for CompileError {
         let loc = self.location;
         if let Some(ref stmt) = self.statement {
             // visualize the error when location and statement are provided
-            write!(
-                f,
-                "{}",
-                loc.visualize(stmt, &format_args!("{} at {}", self.error, loc))
-            )
+            loc.fmt_with(f, &self.error)?;
+            write!(f, "\n{stmt}{arrow:>pad$}", pad = loc.column(), arrow = "^")
         } else {
-            write!(f, "{} at {}", self.error, loc)
+            loc.fmt_with(f, &self.error)
         }
     }
 }

From 9d67b944cf7a7a2d581fda3120aa3b5c73142e39 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 09:07:15 +0900
Subject: [PATCH 262/459] Integrate ParseError to compiler-core::Error

---
 core/src/error.rs     | 22 ++++++++++++++
 core/src/lib.rs       |  2 ++
 parser/Cargo.toml     |  1 +
 parser/src/error.rs   | 70 ++++++++++++++++++++++++-------------------
 parser/src/fstring.rs |  4 +--
 src/lib.rs            |  2 ++
 6 files changed, 69 insertions(+), 32 deletions(-)
 create mode 100644 core/src/error.rs

diff --git a/core/src/error.rs b/core/src/error.rs
new file mode 100644
index 00000000..4553b60a
--- /dev/null
+++ b/core/src/error.rs
@@ -0,0 +1,22 @@
+use crate::Location;
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct Error<T> {
+    pub error: T,
+    pub location: Location,
+    pub source_path: String,
+}
+
+impl<T> std::ops::Deref for Error<T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        &self.error
+    }
+}
+
+impl<T> Error<T> {
+    pub fn error(self) -> T {
+        self.error
+    }
+}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 0b2e23e8..f38c6d0b 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -2,9 +2,11 @@
 #![doc(html_root_url = "https://docs.rs/rustpython-compiler-core/")]
 
 mod bytecode;
+mod error;
 mod location;
 mod mode;
 
 pub use bytecode::*;
+pub use error::Error;
 pub use location::Location;
 pub use mode::Mode;
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index 2522e726..e6acbbd4 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -16,6 +16,7 @@ tiny-keccak = { version = "2", features = ["sha3"] }
 
 [dependencies]
 rustpython-ast = { path = "../ast" }
+rustpython-compiler-core = { path = "../core" }
 
 ahash = "0.7.6"
 itertools = "0.10.3"
diff --git a/parser/src/error.rs b/parser/src/error.rs
index 76f02dc5..b64abcb7 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -118,11 +118,7 @@ impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
 
 /// Represents an error during parsing
 #[derive(Debug, PartialEq)]
-pub struct ParseError {
-    pub error: ParseErrorType,
-    pub location: Location,
-    pub source_path: String,
-}
+pub struct ParseError(rustpython_compiler_core::Error<ParseErrorType>);
 
 #[derive(Debug, PartialEq)]
 pub enum ParseErrorType {
@@ -138,8 +134,28 @@ pub enum ParseErrorType {
     Lexical(LexicalErrorType),
 }
 
+impl From<ParseError> for rustpython_compiler_core::Error<ParseErrorType> {
+    fn from(err: ParseError) -> Self {
+        err.0
+    }
+}
+
+impl From<ParseError> for ParseErrorType {
+    fn from(err: ParseError) -> Self {
+        err.0.error
+    }
+}
+
 /// Convert `lalrpop_util::ParseError` to our internal type
 impl ParseError {
+    fn new(error: ParseErrorType, location: Location, source_path: String) -> Self {
+        Self(rustpython_compiler_core::Error {
+            error,
+            location,
+            source_path,
+        })
+    }
+
     pub(crate) fn from_lalrpop(
         err: LalrpopError<Location, Tok, LexicalError>,
         source_path: &str,
@@ -147,36 +163,30 @@ impl ParseError {
         let source_path = source_path.to_owned();
         match err {
             // TODO: Are there cases where this isn't an EOF?
-            LalrpopError::InvalidToken { location } => ParseError {
-                error: ParseErrorType::Eof,
-                location,
-                source_path,
-            },
-            LalrpopError::ExtraToken { token } => ParseError {
-                error: ParseErrorType::ExtraToken(token.1),
-                location: token.0,
-                source_path,
-            },
-            LalrpopError::User { error } => ParseError {
-                error: ParseErrorType::Lexical(error.error),
-                location: error.location,
+            LalrpopError::InvalidToken { location } => {
+                ParseError::new(ParseErrorType::Eof, location, source_path)
+            }
+            LalrpopError::ExtraToken { token } => {
+                ParseError::new(ParseErrorType::ExtraToken(token.1), token.0, source_path)
+            }
+            LalrpopError::User { error } => ParseError::new(
+                ParseErrorType::Lexical(error.error),
+                error.location,
                 source_path,
-            },
+            ),
             LalrpopError::UnrecognizedToken { token, expected } => {
                 // Hacky, but it's how CPython does it. See PyParser_AddToken,
                 // in particular "Only one possible expected token" comment.
                 let expected = (expected.len() == 1).then(|| expected[0].clone());
-                ParseError {
-                    error: ParseErrorType::UnrecognizedToken(token.1, expected),
-                    location: token.0,
+                ParseError::new(
+                    ParseErrorType::UnrecognizedToken(token.1, expected),
+                    token.0,
                     source_path,
-                }
+                )
+            }
+            LalrpopError::UnrecognizedEOF { location, .. } => {
+                ParseError::new(ParseErrorType::Eof, location, source_path)
             }
-            LalrpopError::UnrecognizedEOF { location, .. } => ParseError {
-                error: ParseErrorType::Eof,
-                location,
-                source_path,
-            },
         }
     }
 }
@@ -229,9 +239,9 @@ impl ParseErrorType {
 }
 
 impl std::ops::Deref for ParseError {
-    type Target = ParseErrorType;
+    type Target = rustpython_compiler_core::Error<ParseErrorType>;
     fn deref(&self) -> &Self::Target {
-        &self.error
+        &self.0
     }
 }
 
diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index 11d75a09..dc170635 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -186,7 +186,7 @@ impl<'a> FStringParser<'a> {
                         vec![self.expr(ExprKind::FormattedValue {
                             value: Box::new(
                                 parse_fstring_expr(&expression)
-                                    .map_err(|e| InvalidExpression(Box::new(e.error)))?,
+                                    .map_err(|e| InvalidExpression(Box::new(e.into())))?,
                             ),
                             conversion: conversion as _,
                             format_spec: spec,
@@ -204,7 +204,7 @@ impl<'a> FStringParser<'a> {
                             self.expr(ExprKind::FormattedValue {
                                 value: Box::new(
                                     parse_fstring_expr(&expression)
-                                        .map_err(|e| InvalidExpression(Box::new(e.error)))?,
+                                        .map_err(|e| InvalidExpression(Box::new(e.into())))?,
                                 ),
                                 conversion: (if conversion == ConversionFlag::None && spec.is_none()
                                 {
diff --git a/src/lib.rs b/src/lib.rs
index a96eefa4..4e057a67 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -2,6 +2,7 @@ use rustpython_codegen::{compile, symboltable};
 use rustpython_compiler_core::CodeObject;
 use rustpython_parser::{
     ast::{fold::Fold, ConstantOptimizer, Location},
+    error::ParseErrorType,
     parser,
 };
 use std::fmt;
@@ -48,6 +49,7 @@ impl CompileError {
         }
     }
     fn from_parse(error: rustpython_parser::error::ParseError, source: &str) -> Self {
+        let error: rustpython_compiler_core::Error<ParseErrorType> = error.into();
         Self {
             error: error.error.into(),
             location: error.location,

From 1192a11d394190213060ae6475561f03597da393 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 18:38:55 +0900
Subject: [PATCH 263/459] integrate CodegenError to compiler-core::Error

---
 codegen/src/error.rs | 20 +-------------------
 core/src/error.rs    | 20 ++++++++++++++++++++
 parser/src/error.rs  |  2 +-
 src/lib.rs           |  4 ++--
 4 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/codegen/src/error.rs b/codegen/src/error.rs
index 8c3ea3f2..a4ac60bd 100644
--- a/codegen/src/error.rs
+++ b/codegen/src/error.rs
@@ -1,12 +1,6 @@
-use rustpython_compiler_core::Location;
 use std::{error::Error, fmt};
 
-#[derive(Debug)]
-pub struct CodegenError {
-    pub error: CodegenErrorType,
-    pub location: Location,
-    pub source_path: String,
-}
+pub type CodegenError = rustpython_compiler_core::Error<CodegenErrorType>;
 
 #[derive(Debug)]
 #[non_exhaustive]
@@ -87,15 +81,3 @@ impl fmt::Display for CodegenErrorType {
 }
 
 impl Error for CodegenErrorType {}
-
-impl fmt::Display for CodegenError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        self.location.fmt_with(f, &self.error)
-    }
-}
-
-impl Error for CodegenError {
-    fn source(&self) -> Option<&(dyn Error + 'static)> {
-        None
-    }
-}
diff --git a/core/src/error.rs b/core/src/error.rs
index 4553b60a..d2497555 100644
--- a/core/src/error.rs
+++ b/core/src/error.rs
@@ -1,3 +1,5 @@
+use std::fmt::Display;
+
 use crate::Location;
 
 #[derive(Debug, PartialEq, Eq)]
@@ -15,6 +17,24 @@ impl<T> std::ops::Deref for Error<T> {
     }
 }
 
+impl<T> std::error::Error for Error<T>
+where
+    T: std::fmt::Display + std::fmt::Debug,
+{
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        None
+    }
+}
+
+impl<T> Display for Error<T>
+where
+    T: std::fmt::Display,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        self.location.fmt_with(f, &self.error)
+    }
+}
+
 impl<T> Error<T> {
     pub fn error(self) -> T {
         self.error
diff --git a/parser/src/error.rs b/parser/src/error.rs
index b64abcb7..257d4c69 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -193,7 +193,7 @@ impl ParseError {
 
 impl fmt::Display for ParseError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        self.location.fmt_with(f, &self.error)
+        self.0.fmt(f)
     }
 }
 
diff --git a/src/lib.rs b/src/lib.rs
index 4e057a67..933f0d02 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -11,7 +11,7 @@ pub use rustpython_codegen::compile::CompileOpts;
 pub use rustpython_compiler_core::Mode;
 
 #[derive(Debug, thiserror::Error)]
-pub enum CodegenErrorType {
+pub enum CompileErrorType {
     #[error(transparent)]
     Compile(#[from] rustpython_codegen::error::CodegenErrorType),
     #[error(transparent)]
@@ -20,7 +20,7 @@ pub enum CodegenErrorType {
 
 #[derive(Debug, thiserror::Error)]
 pub struct CompileError {
-    pub error: CodegenErrorType,
+    pub error: CompileErrorType,
     pub source_path: String,
     pub location: Location,
     pub statement: Option<String>,

From 5b50c547d6132a7ebaa9d344f65becbe129f13ce Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 22 Aug 2022 20:21:40 +0900
Subject: [PATCH 264/459] Integrate CompileError to compiler-core::BaseError

---
 codegen/src/error.rs |  2 +-
 core/src/error.rs    | 28 +++++++++++++++++++++++-----
 core/src/lib.rs      |  2 +-
 parser/src/error.rs  |  8 ++++----
 src/lib.rs           | 31 ++++++++++++++++++-------------
 5 files changed, 47 insertions(+), 24 deletions(-)

diff --git a/codegen/src/error.rs b/codegen/src/error.rs
index a4ac60bd..cfc4741c 100644
--- a/codegen/src/error.rs
+++ b/codegen/src/error.rs
@@ -1,6 +1,6 @@
 use std::{error::Error, fmt};
 
-pub type CodegenError = rustpython_compiler_core::Error<CodegenErrorType>;
+pub type CodegenError = rustpython_compiler_core::BaseError<CodegenErrorType>;
 
 #[derive(Debug)]
 #[non_exhaustive]
diff --git a/core/src/error.rs b/core/src/error.rs
index d2497555..40d97343 100644
--- a/core/src/error.rs
+++ b/core/src/error.rs
@@ -3,13 +3,13 @@ use std::fmt::Display;
 use crate::Location;
 
 #[derive(Debug, PartialEq, Eq)]
-pub struct Error<T> {
+pub struct BaseError<T> {
     pub error: T,
     pub location: Location,
     pub source_path: String,
 }
 
-impl<T> std::ops::Deref for Error<T> {
+impl<T> std::ops::Deref for BaseError<T> {
     type Target = T;
 
     fn deref(&self) -> &Self::Target {
@@ -17,7 +17,7 @@ impl<T> std::ops::Deref for Error<T> {
     }
 }
 
-impl<T> std::error::Error for Error<T>
+impl<T> std::error::Error for BaseError<T>
 where
     T: std::fmt::Display + std::fmt::Debug,
 {
@@ -26,7 +26,7 @@ where
     }
 }
 
-impl<T> Display for Error<T>
+impl<T> Display for BaseError<T>
 where
     T: std::fmt::Display,
 {
@@ -35,8 +35,26 @@ where
     }
 }
 
-impl<T> Error<T> {
+impl<T> BaseError<T> {
     pub fn error(self) -> T {
         self.error
     }
+
+    pub fn from<U>(obj: BaseError<U>) -> Self
+    where
+        U: Into<T>,
+    {
+        Self {
+            error: obj.error.into(),
+            location: obj.location,
+            source_path: obj.source_path,
+        }
+    }
+
+    pub fn into<U>(self) -> BaseError<U>
+    where
+        T: Into<U>,
+    {
+        BaseError::from(self)
+    }
 }
diff --git a/core/src/lib.rs b/core/src/lib.rs
index f38c6d0b..7360cf77 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -7,6 +7,6 @@ mod location;
 mod mode;
 
 pub use bytecode::*;
-pub use error::Error;
+pub use error::BaseError;
 pub use location::Location;
 pub use mode::Mode;
diff --git a/parser/src/error.rs b/parser/src/error.rs
index 257d4c69..77f2f8f8 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -118,7 +118,7 @@ impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
 
 /// Represents an error during parsing
 #[derive(Debug, PartialEq)]
-pub struct ParseError(rustpython_compiler_core::Error<ParseErrorType>);
+pub struct ParseError(rustpython_compiler_core::BaseError<ParseErrorType>);
 
 #[derive(Debug, PartialEq)]
 pub enum ParseErrorType {
@@ -134,7 +134,7 @@ pub enum ParseErrorType {
     Lexical(LexicalErrorType),
 }
 
-impl From<ParseError> for rustpython_compiler_core::Error<ParseErrorType> {
+impl From<ParseError> for rustpython_compiler_core::BaseError<ParseErrorType> {
     fn from(err: ParseError) -> Self {
         err.0
     }
@@ -149,7 +149,7 @@ impl From<ParseError> for ParseErrorType {
 /// Convert `lalrpop_util::ParseError` to our internal type
 impl ParseError {
     fn new(error: ParseErrorType, location: Location, source_path: String) -> Self {
-        Self(rustpython_compiler_core::Error {
+        Self(rustpython_compiler_core::BaseError {
             error,
             location,
             source_path,
@@ -239,7 +239,7 @@ impl ParseErrorType {
 }
 
 impl std::ops::Deref for ParseError {
-    type Target = rustpython_compiler_core::Error<ParseErrorType>;
+    type Target = rustpython_compiler_core::BaseError<ParseErrorType>;
     fn deref(&self) -> &Self::Target {
         &self.0
     }
diff --git a/src/lib.rs b/src/lib.rs
index 933f0d02..afcb1adc 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,5 +1,5 @@
 use rustpython_codegen::{compile, symboltable};
-use rustpython_compiler_core::CodeObject;
+use rustpython_compiler_core::{BaseError, CodeObject};
 use rustpython_parser::{
     ast::{fold::Fold, ConstantOptimizer, Location},
     error::ParseErrorType,
@@ -18,14 +18,21 @@ pub enum CompileErrorType {
     Parse(#[from] rustpython_parser::error::ParseErrorType),
 }
 
+pub type CompileErrorBody = BaseError<CompileErrorType>;
+
 #[derive(Debug, thiserror::Error)]
 pub struct CompileError {
-    pub error: CompileErrorType,
-    pub source_path: String,
-    pub location: Location,
+    pub body: CompileErrorBody,
     pub statement: Option<String>,
 }
 
+impl std::ops::Deref for CompileError {
+    type Target = CompileErrorBody;
+    fn deref(&self) -> &Self::Target {
+        &self.body
+    }
+}
+
 impl fmt::Display for CompileError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         let loc = self.location;
@@ -41,20 +48,18 @@ impl fmt::Display for CompileError {
 
 impl CompileError {
     fn from_codegen(error: rustpython_codegen::error::CodegenError, source: &str) -> Self {
+        let statement = get_statement(source, error.location);
         Self {
-            error: error.error.into(),
-            location: error.location,
-            source_path: error.source_path,
-            statement: get_statement(source, error.location),
+            body: error.into(),
+            statement,
         }
     }
     fn from_parse(error: rustpython_parser::error::ParseError, source: &str) -> Self {
-        let error: rustpython_compiler_core::Error<ParseErrorType> = error.into();
+        let error: rustpython_compiler_core::BaseError<ParseErrorType> = error.into();
+        let statement = get_statement(source, error.location);
         Self {
-            error: error.error.into(),
-            location: error.location,
-            source_path: error.source_path,
-            statement: get_statement(source, error.location),
+            body: error.into(),
+            statement,
         }
     }
     fn from_symtable(

From 4724cc63f72241c4124e0515f0d5367bebfd4b21 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Tue, 23 Aug 2022 01:30:00 +0900
Subject: [PATCH 265/459] Move out CompileError to core as generic form

---
 core/Cargo.toml   |  1 +
 core/src/error.rs | 29 ++++++++++++++++++
 core/src/lib.rs   |  2 +-
 src/lib.rs        | 75 +++++++++++++----------------------------------
 4 files changed, 52 insertions(+), 55 deletions(-)

diff --git a/core/Cargo.toml b/core/Cargo.toml
index 5821ddc3..65be384f 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -17,3 +17,4 @@ num-bigint = { version = "0.4.3", features = ["serde"] }
 num-complex = { version = "0.4.0", features = ["serde"] }
 serde = { version = "1.0.136", features = ["derive"] }
 static_assertions = "1.1.0"
+thiserror = "1.0"
diff --git a/core/src/error.rs b/core/src/error.rs
index 40d97343..f69b61f0 100644
--- a/core/src/error.rs
+++ b/core/src/error.rs
@@ -58,3 +58,32 @@ impl<T> BaseError<T> {
         BaseError::from(self)
     }
 }
+
+#[derive(Debug, thiserror::Error)]
+pub struct CompileError<T> {
+    pub body: BaseError<T>,
+    pub statement: Option<String>,
+}
+
+impl<T> std::ops::Deref for CompileError<T> {
+    type Target = BaseError<T>;
+    fn deref(&self) -> &Self::Target {
+        &self.body
+    }
+}
+
+impl<T> std::fmt::Display for CompileError<T>
+where
+    T: std::fmt::Display,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        let loc = self.location;
+        if let Some(ref stmt) = self.statement {
+            // visualize the error when location and statement are provided
+            loc.fmt_with(f, &self.error)?;
+            write!(f, "\n{stmt}{arrow:>pad$}", pad = loc.column(), arrow = "^")
+        } else {
+            loc.fmt_with(f, &self.error)
+        }
+    }
+}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 7360cf77..e1f0a5b3 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -7,6 +7,6 @@ mod location;
 mod mode;
 
 pub use bytecode::*;
-pub use error::BaseError;
+pub use error::{BaseError, CompileError};
 pub use location::Location;
 pub use mode::Mode;
diff --git a/src/lib.rs b/src/lib.rs
index afcb1adc..a017c3ba 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,14 +1,13 @@
 use rustpython_codegen::{compile, symboltable};
-use rustpython_compiler_core::{BaseError, CodeObject};
+use rustpython_compiler_core::CodeObject;
 use rustpython_parser::{
     ast::{fold::Fold, ConstantOptimizer, Location},
     error::ParseErrorType,
     parser,
 };
-use std::fmt;
 
 pub use rustpython_codegen::compile::CompileOpts;
-pub use rustpython_compiler_core::Mode;
+pub use rustpython_compiler_core::{BaseError as CompileErrorBody, Mode};
 
 #[derive(Debug, thiserror::Error)]
 pub enum CompileErrorType {
@@ -18,56 +17,25 @@ pub enum CompileErrorType {
     Parse(#[from] rustpython_parser::error::ParseErrorType),
 }
 
-pub type CompileErrorBody = BaseError<CompileErrorType>;
+pub type CompileError = rustpython_compiler_core::CompileError<CompileErrorType>;
 
-#[derive(Debug, thiserror::Error)]
-pub struct CompileError {
-    pub body: CompileErrorBody,
-    pub statement: Option<String>,
-}
-
-impl std::ops::Deref for CompileError {
-    type Target = CompileErrorBody;
-    fn deref(&self) -> &Self::Target {
-        &self.body
-    }
-}
-
-impl fmt::Display for CompileError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let loc = self.location;
-        if let Some(ref stmt) = self.statement {
-            // visualize the error when location and statement are provided
-            loc.fmt_with(f, &self.error)?;
-            write!(f, "\n{stmt}{arrow:>pad$}", pad = loc.column(), arrow = "^")
-        } else {
-            loc.fmt_with(f, &self.error)
-        }
+fn error_from_codegen(
+    error: rustpython_codegen::error::CodegenError,
+    source: &str,
+) -> CompileError {
+    let statement = get_statement(source, error.location);
+    CompileError {
+        body: error.into(),
+        statement,
     }
 }
 
-impl CompileError {
-    fn from_codegen(error: rustpython_codegen::error::CodegenError, source: &str) -> Self {
-        let statement = get_statement(source, error.location);
-        Self {
-            body: error.into(),
-            statement,
-        }
-    }
-    fn from_parse(error: rustpython_parser::error::ParseError, source: &str) -> Self {
-        let error: rustpython_compiler_core::BaseError<ParseErrorType> = error.into();
-        let statement = get_statement(source, error.location);
-        Self {
-            body: error.into(),
-            statement,
-        }
-    }
-    fn from_symtable(
-        error: symboltable::SymbolTableError,
-        source: &str,
-        source_path: String,
-    ) -> Self {
-        Self::from_codegen(error.into_codegen_error(source_path), source)
+fn error_from_parse(error: rustpython_parser::error::ParseError, source: &str) -> CompileError {
+    let error: CompileErrorBody<ParseErrorType> = error.into();
+    let statement = get_statement(source, error.location);
+    CompileError {
+        body: error.into(),
+        statement,
     }
 }
 
@@ -85,15 +53,14 @@ pub fn compile(
     };
     let mut ast = match parser::parse(source, parser_mode, &source_path) {
         Ok(x) => x,
-        Err(e) => return Err(CompileError::from_parse(e, source)),
+        Err(e) => return Err(error_from_parse(e, source)),
     };
     if opts.optimize > 0 {
         ast = ConstantOptimizer::new()
             .fold_mod(ast)
             .unwrap_or_else(|e| match e {});
     }
-    compile::compile_top(&ast, source_path, mode, opts)
-        .map_err(|e| CompileError::from_codegen(e, source))
+    compile::compile_top(&ast, source_path, mode, opts).map_err(|e| error_from_codegen(e, source))
 }
 
 pub fn compile_symtable(
@@ -101,7 +68,7 @@ pub fn compile_symtable(
     mode: compile::Mode,
     source_path: &str,
 ) -> Result<symboltable::SymbolTable, CompileError> {
-    let parse_err = |e| CompileError::from_parse(e, source);
+    let parse_err = |e| error_from_parse(e, source);
     let res = match mode {
         compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
             let ast = parser::parse_program(source, source_path).map_err(parse_err)?;
@@ -112,7 +79,7 @@ pub fn compile_symtable(
             symboltable::SymbolTable::scan_expr(&expr)
         }
     };
-    res.map_err(|e| CompileError::from_symtable(e, source, source_path.to_owned()))
+    res.map_err(|e| error_from_codegen(e.into_codegen_error(source_path.to_owned()), source))
 }
 
 fn get_statement(source: &str, loc: Location) -> Option<String> {

From a1640e4aa1e028ad01704658532845c2b15f95b8 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Tue, 23 Aug 2022 02:04:56 +0900
Subject: [PATCH 266/459] new_syntax_error working without full compiler

Fix #4100
---
 codegen/Cargo.toml    |   1 +
 codegen/src/error.rs  |   6 +--
 core/src/error.rs     |  24 +++++++++-
 parser/src/error.rs   | 102 ++++++++++++++----------------------------
 parser/src/fstring.rs |   4 +-
 parser/src/parser.rs  |   2 +-
 src/lib.rs            |  33 +++-----------
 7 files changed, 66 insertions(+), 106 deletions(-)

diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml
index 424d85a6..3d1a455d 100644
--- a/codegen/Cargo.toml
+++ b/codegen/Cargo.toml
@@ -17,6 +17,7 @@ itertools = "0.10.3"
 log = "0.4.16"
 num-complex = { version = "0.4.0", features = ["serde"] }
 num-traits = "0.2.14"
+thiserror = "1.0"
 
 [dev-dependencies]
 rustpython-parser = { path = "../parser" }
diff --git a/codegen/src/error.rs b/codegen/src/error.rs
index cfc4741c..9073abb5 100644
--- a/codegen/src/error.rs
+++ b/codegen/src/error.rs
@@ -1,8 +1,8 @@
-use std::{error::Error, fmt};
+use std::fmt;
 
 pub type CodegenError = rustpython_compiler_core::BaseError<CodegenErrorType>;
 
-#[derive(Debug)]
+#[derive(Debug, thiserror::Error)]
 #[non_exhaustive]
 pub enum CodegenErrorType {
     /// Invalid assignment, cannot store value in target.
@@ -79,5 +79,3 @@ impl fmt::Display for CodegenErrorType {
         }
     }
 }
-
-impl Error for CodegenErrorType {}
diff --git a/core/src/error.rs b/core/src/error.rs
index f69b61f0..fde16d88 100644
--- a/core/src/error.rs
+++ b/core/src/error.rs
@@ -1,6 +1,5 @@
-use std::fmt::Display;
-
 use crate::Location;
+use std::fmt::Display;
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct BaseError<T> {
@@ -87,3 +86,24 @@ where
         }
     }
 }
+
+impl<T> CompileError<T> {
+    pub fn from<U>(error: BaseError<U>, source: &str) -> Self
+    where
+        T: From<U>,
+    {
+        let statement = get_statement(source, error.location);
+        CompileError {
+            body: error.into(),
+            statement,
+        }
+    }
+}
+
+fn get_statement(source: &str, loc: Location) -> Option<String> {
+    if loc.column() == 0 || loc.row() == 0 {
+        return None;
+    }
+    let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
+    Some(line + "\n")
+}
diff --git a/parser/src/error.rs b/parser/src/error.rs
index 77f2f8f8..4dd899e1 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -117,8 +117,7 @@ impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
 }
 
 /// Represents an error during parsing
-#[derive(Debug, PartialEq)]
-pub struct ParseError(rustpython_compiler_core::BaseError<ParseErrorType>);
+pub type ParseError = rustpython_compiler_core::BaseError<ParseErrorType>;
 
 #[derive(Debug, PartialEq)]
 pub enum ParseErrorType {
@@ -134,66 +133,44 @@ pub enum ParseErrorType {
     Lexical(LexicalErrorType),
 }
 
-impl From<ParseError> for rustpython_compiler_core::BaseError<ParseErrorType> {
-    fn from(err: ParseError) -> Self {
-        err.0
-    }
-}
-
-impl From<ParseError> for ParseErrorType {
-    fn from(err: ParseError) -> Self {
-        err.0.error
-    }
-}
-
 /// Convert `lalrpop_util::ParseError` to our internal type
-impl ParseError {
-    fn new(error: ParseErrorType, location: Location, source_path: String) -> Self {
-        Self(rustpython_compiler_core::BaseError {
-            error,
+pub(crate) fn parse_error_from_lalrpop(
+    err: LalrpopError<Location, Tok, LexicalError>,
+    source_path: &str,
+) -> ParseError {
+    let source_path = source_path.to_owned();
+    match err {
+        // TODO: Are there cases where this isn't an EOF?
+        LalrpopError::InvalidToken { location } => ParseError {
+            error: ParseErrorType::Eof,
             location,
             source_path,
-        })
-    }
-
-    pub(crate) fn from_lalrpop(
-        err: LalrpopError<Location, Tok, LexicalError>,
-        source_path: &str,
-    ) -> Self {
-        let source_path = source_path.to_owned();
-        match err {
-            // TODO: Are there cases where this isn't an EOF?
-            LalrpopError::InvalidToken { location } => {
-                ParseError::new(ParseErrorType::Eof, location, source_path)
-            }
-            LalrpopError::ExtraToken { token } => {
-                ParseError::new(ParseErrorType::ExtraToken(token.1), token.0, source_path)
-            }
-            LalrpopError::User { error } => ParseError::new(
-                ParseErrorType::Lexical(error.error),
-                error.location,
+        },
+        LalrpopError::ExtraToken { token } => ParseError {
+            error: ParseErrorType::ExtraToken(token.1),
+            location: token.0,
+            source_path,
+        },
+        LalrpopError::User { error } => ParseError {
+            error: ParseErrorType::Lexical(error.error),
+            location: error.location,
+            source_path,
+        },
+        LalrpopError::UnrecognizedToken { token, expected } => {
+            // Hacky, but it's how CPython does it. See PyParser_AddToken,
+            // in particular "Only one possible expected token" comment.
+            let expected = (expected.len() == 1).then(|| expected[0].clone());
+            ParseError {
+                error: ParseErrorType::UnrecognizedToken(token.1, expected),
+                location: token.0,
                 source_path,
-            ),
-            LalrpopError::UnrecognizedToken { token, expected } => {
-                // Hacky, but it's how CPython does it. See PyParser_AddToken,
-                // in particular "Only one possible expected token" comment.
-                let expected = (expected.len() == 1).then(|| expected[0].clone());
-                ParseError::new(
-                    ParseErrorType::UnrecognizedToken(token.1, expected),
-                    token.0,
-                    source_path,
-                )
-            }
-            LalrpopError::UnrecognizedEOF { location, .. } => {
-                ParseError::new(ParseErrorType::Eof, location, source_path)
             }
         }
-    }
-}
-
-impl fmt::Display for ParseError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        self.0.fmt(f)
+        LalrpopError::UnrecognizedEOF { location, .. } => ParseError {
+            error: ParseErrorType::Eof,
+            location,
+            source_path,
+        },
     }
 }
 
@@ -237,16 +214,3 @@ impl ParseErrorType {
         )
     }
 }
-
-impl std::ops::Deref for ParseError {
-    type Target = rustpython_compiler_core::BaseError<ParseErrorType>;
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl Error for ParseError {
-    fn source(&self) -> Option<&(dyn Error + 'static)> {
-        None
-    }
-}
diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index dc170635..11d75a09 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -186,7 +186,7 @@ impl<'a> FStringParser<'a> {
                         vec![self.expr(ExprKind::FormattedValue {
                             value: Box::new(
                                 parse_fstring_expr(&expression)
-                                    .map_err(|e| InvalidExpression(Box::new(e.into())))?,
+                                    .map_err(|e| InvalidExpression(Box::new(e.error)))?,
                             ),
                             conversion: conversion as _,
                             format_spec: spec,
@@ -204,7 +204,7 @@ impl<'a> FStringParser<'a> {
                             self.expr(ExprKind::FormattedValue {
                                 value: Box::new(
                                     parse_fstring_expr(&expression)
-                                        .map_err(|e| InvalidExpression(Box::new(e.into())))?,
+                                        .map_err(|e| InvalidExpression(Box::new(e.error)))?,
                                 ),
                                 conversion: (if conversion == ConversionFlag::None && spec.is_none()
                                 {
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index ff86ef40..cd74856c 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -74,7 +74,7 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
 
     python::TopParser::new()
         .parse(tokenizer)
-        .map_err(|e| ParseError::from_lalrpop(e, source_path))
+        .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
 }
 
 #[cfg(test)]
diff --git a/src/lib.rs b/src/lib.rs
index a017c3ba..4d318a10 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,7 +1,7 @@
 use rustpython_codegen::{compile, symboltable};
 use rustpython_compiler_core::CodeObject;
 use rustpython_parser::{
-    ast::{fold::Fold, ConstantOptimizer, Location},
+    ast::{fold::Fold, ConstantOptimizer},
     error::ParseErrorType,
     parser,
 };
@@ -12,31 +12,16 @@ pub use rustpython_compiler_core::{BaseError as CompileErrorBody, Mode};
 #[derive(Debug, thiserror::Error)]
 pub enum CompileErrorType {
     #[error(transparent)]
-    Compile(#[from] rustpython_codegen::error::CodegenErrorType),
+    Codegen(#[from] rustpython_codegen::error::CodegenErrorType),
     #[error(transparent)]
     Parse(#[from] rustpython_parser::error::ParseErrorType),
 }
 
 pub type CompileError = rustpython_compiler_core::CompileError<CompileErrorType>;
 
-fn error_from_codegen(
-    error: rustpython_codegen::error::CodegenError,
-    source: &str,
-) -> CompileError {
-    let statement = get_statement(source, error.location);
-    CompileError {
-        body: error.into(),
-        statement,
-    }
-}
-
 fn error_from_parse(error: rustpython_parser::error::ParseError, source: &str) -> CompileError {
     let error: CompileErrorBody<ParseErrorType> = error.into();
-    let statement = get_statement(source, error.location);
-    CompileError {
-        body: error.into(),
-        statement,
-    }
+    CompileError::from(error, source)
 }
 
 /// Compile a given sourcecode into a bytecode object.
@@ -60,7 +45,7 @@ pub fn compile(
             .fold_mod(ast)
             .unwrap_or_else(|e| match e {});
     }
-    compile::compile_top(&ast, source_path, mode, opts).map_err(|e| error_from_codegen(e, source))
+    compile::compile_top(&ast, source_path, mode, opts).map_err(|e| CompileError::from(e, source))
 }
 
 pub fn compile_symtable(
@@ -79,13 +64,5 @@ pub fn compile_symtable(
             symboltable::SymbolTable::scan_expr(&expr)
         }
     };
-    res.map_err(|e| error_from_codegen(e.into_codegen_error(source_path.to_owned()), source))
-}
-
-fn get_statement(source: &str, loc: Location) -> Option<String> {
-    if loc.column() == 0 || loc.row() == 0 {
-        return None;
-    }
-    let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
-    Some(line + "\n")
+    res.map_err(|e| CompileError::from(e.into_codegen_error(source_path.to_owned()), source))
 }

From 122382cb53fb91a3db4cf601c30284d6f2a5c5de Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Tue, 23 Aug 2022 03:59:51 +0900
Subject: [PATCH 267/459] use thiserror

---
 core/src/error.rs   | 9 +--------
 parser/Cargo.toml   | 1 +
 parser/src/error.rs | 6 ++----
 3 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/core/src/error.rs b/core/src/error.rs
index fde16d88..9619e1d8 100644
--- a/core/src/error.rs
+++ b/core/src/error.rs
@@ -16,14 +16,7 @@ impl<T> std::ops::Deref for BaseError<T> {
     }
 }
 
-impl<T> std::error::Error for BaseError<T>
-where
-    T: std::fmt::Display + std::fmt::Debug,
-{
-    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
-        None
-    }
-}
+impl<T> std::error::Error for BaseError<T> where T: std::fmt::Display + std::fmt::Debug {}
 
 impl<T> Display for BaseError<T>
 where
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index e6acbbd4..3b5fd3b1 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -28,6 +28,7 @@ phf = "0.10.1"
 unic-emoji-char = "0.9.0"
 unic-ucd-ident  = "0.9.0"
 unicode_names2 = "0.5.0"
+thiserror = "1.0"
 
 [dev-dependencies]
 insta = "1.14.0"
diff --git a/parser/src/error.rs b/parser/src/error.rs
index 4dd899e1..19f885d4 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -3,7 +3,7 @@
 
 use crate::{ast::Location, token::Tok};
 use lalrpop_util::ParseError as LalrpopError;
-use std::{error::Error, fmt};
+use std::fmt;
 
 /// Represents an error during lexical scanning.
 #[derive(Debug, PartialEq)]
@@ -119,7 +119,7 @@ impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
 /// Represents an error during parsing
 pub type ParseError = rustpython_compiler_core::BaseError<ParseErrorType>;
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, thiserror::Error)]
 pub enum ParseErrorType {
     /// Parser encountered an unexpected end of input
     Eof,
@@ -194,8 +194,6 @@ impl fmt::Display for ParseErrorType {
     }
 }
 
-impl Error for ParseErrorType {}
-
 impl ParseErrorType {
     pub fn is_indentation_error(&self) -> bool {
         match self {

From 8d39d5362b52360310eea22661ef5de74655a4e7 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Tue, 23 Aug 2022 04:17:06 +0900
Subject: [PATCH 268/459] parser::Mode from compile::Mode

---
 parser/src/mode.rs | 11 +++++++++++
 src/lib.rs         |  7 +------
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/parser/src/mode.rs b/parser/src/mode.rs
index c03a538e..cd84a098 100644
--- a/parser/src/mode.rs
+++ b/parser/src/mode.rs
@@ -17,6 +17,17 @@ impl Mode {
     }
 }
 
+impl From<rustpython_compiler_core::Mode> for Mode {
+    fn from(mode: rustpython_compiler_core::Mode) -> Self {
+        use rustpython_compiler_core::Mode as CompileMode;
+        match mode {
+            CompileMode::Exec => Self::Module,
+            CompileMode::Eval => Self::Expression,
+            CompileMode::Single | CompileMode::BlockExpr => Self::Interactive,
+        }
+    }
+}
+
 impl std::str::FromStr for Mode {
     type Err = ModeParseError;
     fn from_str(s: &str) -> Result<Self, ModeParseError> {
diff --git a/src/lib.rs b/src/lib.rs
index 4d318a10..c488a400 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -31,12 +31,7 @@ pub fn compile(
     source_path: String,
     opts: compile::CompileOpts,
 ) -> Result<CodeObject, CompileError> {
-    let parser_mode = match mode {
-        compile::Mode::Exec => parser::Mode::Module,
-        compile::Mode::Eval => parser::Mode::Expression,
-        compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive,
-    };
-    let mut ast = match parser::parse(source, parser_mode, &source_path) {
+    let mut ast = match parser::parse(source, mode.into(), &source_path) {
         Ok(x) => x,
         Err(e) => return Err(error_from_parse(e, source)),
     };

From 4de373551e9d49ddc1e420ea06485c9710eb4312 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Tue, 23 Aug 2022 05:15:27 +0900
Subject: [PATCH 269/459] remove Mode from codegen root

---
 ast/src/fold_helpers.rs | 3 +--
 codegen/src/lib.rs      | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/ast/src/fold_helpers.rs b/ast/src/fold_helpers.rs
index f15a9d5d..969ea4e5 100644
--- a/ast/src/fold_helpers.rs
+++ b/ast/src/fold_helpers.rs
@@ -1,5 +1,4 @@
-use crate::constant;
-use crate::fold::Fold;
+use crate::{constant, fold::Fold};
 
 pub(crate) trait Foldable<T, U> {
     type Mapped;
diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs
index 9114a049..910d015a 100644
--- a/codegen/src/lib.rs
+++ b/codegen/src/lib.rs
@@ -13,4 +13,4 @@ pub mod error;
 pub mod ir;
 pub mod symboltable;
 
-pub use compile::{CompileOpts, Mode};
+pub use compile::CompileOpts;

From 307bc46d744e696dc09c1ad72118c454d4000b82 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Sun, 4 Sep 2022 16:31:34 -0400
Subject: [PATCH 270/459] Add PartialOrd to Location

---
 core/src/location.rs | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/core/src/location.rs b/core/src/location.rs
index fedcdcf7..73d6ec89 100644
--- a/core/src/location.rs
+++ b/core/src/location.rs
@@ -1,7 +1,7 @@
 use serde::{Deserialize, Serialize};
 
 /// Sourcecode location.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
 pub struct Location {
     pub(super) row: u32,
     pub(super) column: u32,
@@ -59,3 +59,22 @@ impl Location {
         self.column = 1;
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::Location;
+
+    #[test]
+    fn test_gt() {
+        assert!(Location::new(1, 2) > Location::new(1, 1));
+        assert!(Location::new(2, 1) > Location::new(1, 1));
+        assert!(Location::new(2, 1) > Location::new(1, 2));
+    }
+
+    #[test]
+    fn test_lt() {
+        assert!(Location::new(1, 1) < Location::new(1, 2));
+        assert!(Location::new(1, 1) < Location::new(2, 1));
+        assert!(Location::new(1, 2) < Location::new(2, 1));
+    }
+}

From f5ce4a6cc391108fd4a1b9861c60fdfd43652cd5 Mon Sep 17 00:00:00 2001
From: Gyubong <jopemachine@naver.com>
Date: Wed, 21 Sep 2022 16:11:29 +0900
Subject: [PATCH 271/459] Always insert `None` at `code_stack.constants`

---
 codegen/src/compile.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 6cf33acc..2dab12e0 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -1107,6 +1107,10 @@ impl Compiler {
 
         let (doc_str, body) = split_doc(body);
 
+        self.current_codeinfo()
+            .constants
+            .insert_full(ConstantData::None);
+
         self.compile_statements(body)?;
 
         // Emit None at end:

From 2bef1f02d222ce2ac03986ec175fcea8ffcf46b4 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 21 Sep 2022 16:47:06 +0900
Subject: [PATCH 272/459] better parser build experience

---
 parser/build.rs | 50 +++++++++++++++++++++++++++++--------------------
 1 file changed, 30 insertions(+), 20 deletions(-)

diff --git a/parser/build.rs b/parser/build.rs
index 874b0166..81c28f43 100644
--- a/parser/build.rs
+++ b/parser/build.rs
@@ -16,22 +16,26 @@ fn main() -> anyhow::Result<()> {
     Ok(())
 }
 
-fn requires_lalrpop(source: &str, target: &str) -> bool {
+fn requires_lalrpop(source: &str, target: &str) -> Option<String> {
     let target = if let Ok(target) = File::open(target) {
         target
     } else {
-        println!("cargo:warning=python.rs doesn't exist. regenerate.");
-        return true;
+        return Some("python.rs doesn't exist. regenerate.".to_owned());
     };
 
     let sha_prefix = "// sha3: ";
-    let sha3_line = BufReader::with_capacity(128, target)
-        .lines()
-        .find_map(|line| {
-            let line = line.unwrap();
-            line.starts_with(sha_prefix).then_some(line)
-        })
-        .expect("no sha3 line?");
+    let sha3_line = if let Some(sha3_line) =
+        BufReader::with_capacity(128, target)
+            .lines()
+            .find_map(|line| {
+                let line = line.unwrap();
+                line.starts_with(sha_prefix).then_some(line)
+            }) {
+        sha3_line
+    } else {
+        // no sha3 line - maybe old version of lalrpop installed
+        return Some("python.rs doesn't include sha3 hash. regenerate.".to_owned());
+    };
     let expected_sha3_str = sha3_line.strip_prefix(sha_prefix).unwrap();
 
     let actual_sha3 = {
@@ -55,29 +59,35 @@ fn requires_lalrpop(source: &str, target: &str) -> bool {
     };
     let eq = sha_equal(expected_sha3_str, &actual_sha3);
     if !eq {
-        println!("cargo:warning=python.rs hash expected: {expected_sha3_str}");
         let mut actual_sha3_str = String::new();
         for byte in actual_sha3 {
             write!(actual_sha3_str, "{byte:02x}").unwrap();
         }
-        println!("cargo:warning=python.rs hash   actual: {actual_sha3_str}");
+        return Some(format!(
+            "python.rs hash expected: {expected_sha3_str} but actual: {actual_sha3_str}"
+        ));
     }
-    !eq
+    None
 }
 
 fn try_lalrpop(source: &str, target: &str) -> anyhow::Result<()> {
-    if !requires_lalrpop(source, target) {
+    let _message = if let Some(msg) = requires_lalrpop(source, target) {
+        msg
+    } else {
         return Ok(());
-    }
+    };
 
     #[cfg(feature = "lalrpop")]
-    {
-        lalrpop::process_root().expect("running lalrpop failed");
-        Ok(())
-    }
+    lalrpop::process_root().unwrap_or_else(|e| {
+        println!("cargo:warning={_message}");
+        panic!("running lalrpop failed. {e:?}");
+    });
 
     #[cfg(not(feature = "lalrpop"))]
-    panic!("try: cargo build --manifest-path=compiler/parser/Cargo.toml --features=lalrpop");
+    {
+        println!("cargo:warning=try: cargo build --manifest-path=compiler/parser/Cargo.toml --features=lalrpop");
+    }
+    Ok(())
 }
 
 fn sha_equal(expected_sha3_str: &str, actual_sha3: &[u8; 32]) -> bool {

From cbfcc25ea9e4c9d16c2a9e47083455f6157dae35 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 21 Sep 2022 16:49:35 +0900
Subject: [PATCH 273/459] temporary fix of parser build

---
 parser/Cargo.toml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index 3b5fd3b1..b27bb500 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -8,6 +8,9 @@ repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 edition = "2021"
 
+[features]
+default = ["lalrpop"]  # removing this causes potential build failure
+
 [build-dependencies]
 anyhow = "1.0.45"
 lalrpop = { version = "0.19.8", optional = true }

From abec5c0e1253da9b4fbd66c8e48b3199108cfaf2 Mon Sep 17 00:00:00 2001
From: Anders Kaseorg <andersk@mit.edu>
Date: Wed, 28 Sep 2022 17:58:30 -0700
Subject: [PATCH 274/459] =?UTF-8?q?Spell=20=E2=80=9Carithmetic=E2=80=9D=20?=
 =?UTF-8?q?correctly?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Anders Kaseorg <andersk@mit.edu>
---
 parser/python.lalrpop | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index d72dcc9f..224450e0 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -816,12 +816,12 @@ AndExpression: ast::Expr = {
 };
 
 ShiftExpression: ast::Expr = {
-    <e1:ShiftExpression> <location:@L> <op:ShiftOp> <e2:ArithmaticExpression> => ast::Expr {
+    <e1:ShiftExpression> <location:@L> <op:ShiftOp> <e2:ArithmeticExpression> => ast::Expr {
         location,
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) }
     },
-    ArithmaticExpression,
+    ArithmeticExpression,
 };
 
 ShiftOp: ast::Operator = {
@@ -829,8 +829,8 @@ ShiftOp: ast::Operator = {
     ">>" => ast::Operator::RShift,
 };
 
-ArithmaticExpression: ast::Expr = {
-    <a:ArithmaticExpression> <location:@L> <op:AddOp> <b:Term> => ast::Expr {
+ArithmeticExpression: ast::Expr = {
+    <a:ArithmeticExpression> <location:@L> <op:AddOp> <b:Term> => ast::Expr {
         location,
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }

From 830f20837174c5fd32c15e5cf5780588b7f08f36 Mon Sep 17 00:00:00 2001
From: Gyubong <jopemachine@naver.com>
Date: Sat, 1 Oct 2022 18:23:11 +0900
Subject: [PATCH 275/459] Handle Lambda Expression

---
 codegen/src/compile.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 2dab12e0..ebfd454a 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -2126,6 +2126,11 @@ impl Compiler {
 
                 let name = "<lambda>".to_owned();
                 let mut funcflags = self.enter_function(&name, args)?;
+
+                self.current_codeinfo()
+                    .constants
+                    .insert_full(ConstantData::None);
+
                 self.compile_expression(body)?;
                 self.emit(Instruction::ReturnValue);
                 let code = self.pop_code_object();

From eede189959833012cb7d38749aa0f4fb35c977f9 Mon Sep 17 00:00:00 2001
From: dvermd <315743+dvermd@users.noreply.github.com>
Date: Fri, 14 Oct 2022 05:16:34 +0200
Subject: [PATCH 276/459] Refactor fstrings (#4188)

---
 parser/src/fstring.rs                         | 242 +++++++++---------
 ...ing__tests__parse_fstring_nested_spec.snap |   2 +-
 2 files changed, 126 insertions(+), 118 deletions(-)

diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index 146439db..342e47de 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -4,22 +4,15 @@ use crate::{
     error::{FStringError, FStringErrorType, ParseError},
     parser::parse_expression,
 };
-use itertools::Itertools;
 use std::{iter, mem, str};
 
-struct FStringParser<'a> {
-    chars: iter::Peekable<str::Chars<'a>>,
+struct FStringParser {
     str_location: Location,
-    recurse_lvl: u8,
 }
 
-impl<'a> FStringParser<'a> {
-    fn new(source: &'a str, str_location: Location, recurse_lvl: u8) -> Self {
-        Self {
-            chars: source.chars().peekable(),
-            str_location,
-            recurse_lvl,
-        }
+impl FStringParser {
+    fn new(str_location: Location) -> Self {
+        Self { str_location }
     }
 
     #[inline]
@@ -27,44 +20,48 @@ impl<'a> FStringParser<'a> {
         Expr::new(self.str_location, node)
     }
 
-    fn parse_formatted_value(&mut self) -> Result<Vec<Expr>, FStringErrorType> {
-        let mut expression = String::new();
+    fn parse_formatted_value<'a>(
+        &mut self,
+        mut chars: iter::Peekable<str::Chars<'a>>,
+        nested: u8,
+    ) -> Result<(Vec<Expr>, iter::Peekable<str::Chars<'a>>), FStringErrorType> {
+        let mut expression = String::from("{");
         let mut spec = None;
         let mut delims = Vec::new();
         let mut conversion = ConversionFlag::None;
         let mut self_documenting = false;
         let mut trailing_seq = String::new();
 
-        while let Some(ch) = self.chars.next() {
+        while let Some(ch) = chars.next() {
             match ch {
                 // can be integrated better with the remainign code, but as a starting point ok
                 // in general I would do here a tokenizing of the fstrings to omit this peeking.
-                '!' if self.chars.peek() == Some(&'=') => {
+                '!' if chars.peek() == Some(&'=') => {
                     expression.push_str("!=");
-                    self.chars.next();
+                    chars.next();
                 }
 
-                '=' if self.chars.peek() == Some(&'=') => {
+                '=' if chars.peek() == Some(&'=') => {
                     expression.push_str("==");
-                    self.chars.next();
+                    chars.next();
                 }
 
-                '>' if self.chars.peek() == Some(&'=') => {
+                '>' if chars.peek() == Some(&'=') => {
                     expression.push_str(">=");
-                    self.chars.next();
+                    chars.next();
                 }
 
-                '<' if self.chars.peek() == Some(&'=') => {
+                '<' if chars.peek() == Some(&'=') => {
                     expression.push_str("<=");
-                    self.chars.next();
+                    chars.next();
                 }
 
-                '!' if delims.is_empty() && self.chars.peek() != Some(&'=') => {
+                '!' if delims.is_empty() && chars.peek() != Some(&'=') => {
                     if expression.trim().is_empty() {
                         return Err(EmptyExpression);
                     }
 
-                    conversion = match self.chars.next() {
+                    conversion = match chars.next() {
                         Some('s') => ConversionFlag::Str,
                         Some('a') => ConversionFlag::Ascii,
                         Some('r') => ConversionFlag::Repr,
@@ -76,10 +73,16 @@ impl<'a> FStringParser<'a> {
                         }
                     };
 
-                    if let Some(&peek) = self.chars.peek() {
+                    if let Some(&peek) = chars.peek() {
                         if peek != '}' && peek != ':' {
-                            return Err(ExpectedRbrace);
+                            if expression[1..].trim().is_empty() {
+                                return Err(EmptyExpression);
+                            } else {
+                                return Err(ExpectedRbrace);
+                            }
                         }
+                    } else if expression[1..].trim().is_empty() {
+                        return Err(EmptyExpression);
                     } else {
                         return Err(ExpectedRbrace);
                     }
@@ -87,76 +90,17 @@ impl<'a> FStringParser<'a> {
 
                 // match a python 3.8 self documenting expression
                 // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}'
-                '=' if self.chars.peek() != Some(&'=') && delims.is_empty() => {
+                '=' if chars.peek() != Some(&'=') && delims.is_empty() => {
                     self_documenting = true;
                 }
 
                 ':' if delims.is_empty() => {
-                    let mut nested = 0;
-                    let mut spec_constructor = Vec::new();
-                    let mut constant_piece = String::new();
-                    let mut formatted_value_piece = String::new();
-                    while let Some(&next) = self.chars.peek() {
-                        match next {
-                            '{' if nested > 0 => {
-                                nested += 1;
-                                formatted_value_piece.push(next);
-                            }
-                            '}' if nested > 0 => {
-                                nested -= 1;
-                                if nested == 0 {
-                                    formatted_value_piece.push(next);
-                                    let values = FStringParser::new(
-                                        &formatted_value_piece,
-                                        Location::default(),
-                                        &self.recurse_lvl + 1,
-                                    )
-                                    .parse()?;
-                                    spec_constructor.push(values
-                                        .into_iter()
-                                        .exactly_one()
-                                        .expect("Expected formatted value to produce exactly one expression.")
-                                    );
-                                    formatted_value_piece.clear();
-                                } else {
-                                    formatted_value_piece.push(next);
-                                }
-                            }
-                            _ if nested > 0 => {
-                                formatted_value_piece.push(next);
-                            }
-                            '{' => {
-                                nested += 1;
-                                if !constant_piece.is_empty() {
-                                    spec_constructor.push(self.expr(ExprKind::Constant {
-                                        value: constant_piece.to_owned().into(),
-                                        kind: None,
-                                    }));
-                                    constant_piece.clear();
-                                }
-                                formatted_value_piece.push(next);
-                                formatted_value_piece.push(' ');
-                            }
-                            '}' => break,
-                            _ => {
-                                constant_piece.push(next);
-                            }
-                        }
-                        self.chars.next();
-                    }
-                    if !constant_piece.is_empty() {
-                        spec_constructor.push(self.expr(ExprKind::Constant {
-                            value: constant_piece.to_owned().into(),
-                            kind: None,
-                        }));
-                        constant_piece.clear();
-                    }
-                    if nested > 0 {
-                        return Err(UnclosedLbrace);
-                    }
+                    let (parsed_spec, remaining_chars) = self.parse_spec(chars, nested)?;
+
                     spec = Some(Box::new(self.expr(ExprKind::JoinedStr {
-                        values: spec_constructor,
-                    })))
+                        values: parsed_spec,
+                    })));
+                    chars = remaining_chars;
                 }
                 '(' | '{' | '[' => {
                     expression.push(ch);
@@ -181,13 +125,15 @@ impl<'a> FStringParser<'a> {
                     expression.push(ch);
                 }
                 '}' => {
-                    if expression.is_empty() {
+                    if expression[1..].trim().is_empty() {
                         return Err(EmptyExpression);
                     }
+                    expression.push(ch);
+
                     let ret = if !self_documenting {
                         vec![self.expr(ExprKind::FormattedValue {
                             value: Box::new(
-                                parse_fstring_expr(&expression)
+                                parse_fstring_expr(&expression[1..expression.len() - 1])
                                     .map_err(|e| InvalidExpression(Box::new(e.error)))?,
                             ),
                             conversion: conversion as _,
@@ -196,7 +142,9 @@ impl<'a> FStringParser<'a> {
                     } else {
                         vec![
                             self.expr(ExprKind::Constant {
-                                value: Constant::Str(expression.clone() + "="),
+                                value: Constant::Str(
+                                    expression[1..expression.len() - 1].to_owned() + "=",
+                                ),
                                 kind: None,
                             }),
                             self.expr(ExprKind::Constant {
@@ -205,7 +153,7 @@ impl<'a> FStringParser<'a> {
                             }),
                             self.expr(ExprKind::FormattedValue {
                                 value: Box::new(
-                                    parse_fstring_expr(&expression)
+                                    parse_fstring_expr(&expression[1..expression.len() - 1])
                                         .map_err(|e| InvalidExpression(Box::new(e.error)))?,
                                 ),
                                 conversion: (if conversion == ConversionFlag::None && spec.is_none()
@@ -218,11 +166,11 @@ impl<'a> FStringParser<'a> {
                             }),
                         ]
                     };
-                    return Ok(ret);
+                    return Ok((ret, chars));
                 }
                 '"' | '\'' => {
                     expression.push(ch);
-                    for next in &mut self.chars {
+                    for next in &mut chars {
                         expression.push(next);
                         if next == ch {
                             break;
@@ -236,6 +184,7 @@ impl<'a> FStringParser<'a> {
                     if self_documenting {
                         return Err(ExpectedRbrace);
                     }
+
                     expression.push(ch);
                 }
             }
@@ -243,34 +192,89 @@ impl<'a> FStringParser<'a> {
         Err(UnclosedLbrace)
     }
 
-    fn parse(mut self) -> Result<Vec<Expr>, FStringErrorType> {
-        if self.recurse_lvl >= 2 {
+    fn parse_spec<'a>(
+        &mut self,
+        mut chars: iter::Peekable<str::Chars<'a>>,
+        nested: u8,
+    ) -> Result<(Vec<Expr>, iter::Peekable<str::Chars<'a>>), FStringErrorType> {
+        let mut spec_constructor = Vec::new();
+        let mut constant_piece = String::new();
+        while let Some(&next) = chars.peek() {
+            match next {
+                '{' => {
+                    if !constant_piece.is_empty() {
+                        spec_constructor.push(self.expr(ExprKind::Constant {
+                            value: constant_piece.to_owned().into(),
+                            kind: None,
+                        }));
+                        constant_piece.clear();
+                    }
+                    let (parsed_expr, remaining_chars) = self.parse(chars, nested + 1)?;
+                    spec_constructor.extend(parsed_expr);
+                    chars = remaining_chars;
+                    continue;
+                }
+                '}' => {
+                    break;
+                }
+                _ => {
+                    constant_piece.push(next);
+                }
+            }
+            chars.next();
+        }
+        if !constant_piece.is_empty() {
+            spec_constructor.push(self.expr(ExprKind::Constant {
+                value: constant_piece.to_owned().into(),
+                kind: None,
+            }));
+            constant_piece.clear();
+        }
+        Ok((spec_constructor, chars))
+    }
+
+    fn parse<'a>(
+        &mut self,
+        mut chars: iter::Peekable<str::Chars<'a>>,
+        nested: u8,
+    ) -> Result<(Vec<Expr>, iter::Peekable<str::Chars<'a>>), FStringErrorType> {
+        if nested >= 2 {
             return Err(ExpressionNestedTooDeeply);
         }
 
         let mut content = String::new();
         let mut values = vec![];
 
-        while let Some(ch) = self.chars.next() {
+        while let Some(&ch) = chars.peek() {
             match ch {
                 '{' => {
-                    if let Some('{') = self.chars.peek() {
-                        self.chars.next();
-                        content.push('{');
-                    } else {
-                        if !content.is_empty() {
-                            values.push(self.expr(ExprKind::Constant {
-                                value: mem::take(&mut content).into(),
-                                kind: None,
-                            }));
+                    chars.next();
+                    if nested == 0 {
+                        if let Some('{') = chars.peek() {
+                            chars.next();
+                            content.push('{');
+                            continue;
                         }
-
-                        values.extend(self.parse_formatted_value()?);
                     }
+                    if !content.is_empty() {
+                        values.push(self.expr(ExprKind::Constant {
+                            value: mem::take(&mut content).into(),
+                            kind: None,
+                        }));
+                    }
+
+                    let (parsed_values, remaining_chars) =
+                        self.parse_formatted_value(chars, nested)?;
+                    values.extend(parsed_values);
+                    chars = remaining_chars;
                 }
                 '}' => {
-                    if let Some('}') = self.chars.peek() {
-                        self.chars.next();
+                    if nested > 0 {
+                        break;
+                    }
+                    chars.next();
+                    if let Some('}') = chars.peek() {
+                        chars.next();
                         content.push('}');
                     } else {
                         return Err(UnopenedRbrace);
@@ -278,6 +282,7 @@ impl<'a> FStringParser<'a> {
                 }
                 _ => {
                     content.push(ch);
+                    chars.next();
                 }
             }
         }
@@ -289,7 +294,7 @@ impl<'a> FStringParser<'a> {
             }))
         }
 
-        Ok(values)
+        Ok((values, chars))
     }
 }
 
@@ -301,8 +306,9 @@ fn parse_fstring_expr(source: &str) -> Result<Expr, ParseError> {
 /// Parse an fstring from a string, located at a certain position in the sourcecode.
 /// In case of errors, we will get the location and the error returned.
 pub fn parse_located_fstring(source: &str, location: Location) -> Result<Vec<Expr>, FStringError> {
-    FStringParser::new(source, location, 0)
-        .parse()
+    FStringParser::new(location)
+        .parse(source.chars().peekable(), 0)
+        .map(|(e, _)| e)
         .map_err(|error| FStringError { error, location })
 }
 
@@ -311,7 +317,9 @@ mod tests {
     use super::*;
 
     fn parse_fstring(source: &str) -> Result<Vec<Expr>, FStringErrorType> {
-        FStringParser::new(source, Location::default(), 0).parse()
+        FStringParser::new(Location::default())
+            .parse(source.chars().peekable(), 0)
+            .map(|(e, _)| e)
     }
 
     #[test]
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
index a0262f1a..bb3f26af 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
@@ -41,7 +41,7 @@ expression: parse_ast
                                     value: Located {
                                         location: Location {
                                             row: 1,
-                                            column: 3,
+                                            column: 2,
                                         },
                                         custom: (),
                                         node: Name {

From 15e2ac3fd75ab656e588b74e799d24812a0b071b Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Sat, 15 Oct 2022 11:03:45 -0400
Subject: [PATCH 277/459] Start string location at kind or quote prefix

---
 parser/src/lexer.rs                           |  4 ++-
 ...parser__parser__tests__parse_f_string.snap |  8 ++---
 ...tring__tests__parse_f_string_concat_3.snap |  4 +--
 ..._parse_string_triple_quotes_with_kind.snap | 30 +++++++++++++++++++
 ...ing__tests__parse_u_f_string_concat_1.snap |  8 ++---
 ...ing__tests__parse_u_f_string_concat_2.snap |  8 ++---
 ...tring__tests__parse_u_string_concat_2.snap |  6 ++--
 parser/src/string.rs                          |  7 +++++
 8 files changed, 57 insertions(+), 18 deletions(-)
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 59c76a59..5e5264fe 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -205,7 +205,9 @@ where
 
             // Check if we have a string:
             if self.chr0 == Some('"') || self.chr0 == Some('\'') {
-                return self.lex_string(saw_b, saw_r, saw_u, saw_f);
+                return self
+                    .lex_string(saw_b, saw_r, saw_u, saw_f)
+                    .map(|(_, tok, end_pos)| (start_pos, tok, end_pos));
             }
         }
 
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap
index 445e51dc..97b49bdd 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap
@@ -1,19 +1,19 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
             row: 1,
-            column: 3,
+            column: 1,
         },
         custom: (),
         node: Expr {
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 3,
+                    column: 1,
                 },
                 custom: (),
                 node: JoinedStr {
@@ -21,7 +21,7 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 3,
+                                column: 1,
                             },
                             custom: (),
                             node: Constant {
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
index 197f1c57..a41e7658 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/string.rs
+source: compiler/parser/src/string.rs
 expression: parse_ast
 ---
 [
@@ -34,7 +34,7 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 12,
+                                column: 10,
                             },
                             custom: (),
                             node: FormattedValue {
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap
new file mode 100644
index 00000000..66db2319
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap
@@ -0,0 +1,30 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 1,
+        },
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 1,
+                },
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "Hello, world!",
+                    ),
+                    kind: Some(
+                        "u",
+                    ),
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
index bcba0524..dd6d3388 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
@@ -1,19 +1,19 @@
 ---
-source: parser/src/string.rs
+source: compiler/parser/src/string.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
             row: 1,
-            column: 3,
+            column: 1,
         },
         custom: (),
         node: Expr {
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 3,
+                    column: 1,
                 },
                 custom: (),
                 node: JoinedStr {
@@ -21,7 +21,7 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 3,
+                                column: 1,
                             },
                             custom: (),
                             node: Constant {
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
index 4e4ba23d..b5d6f940 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
@@ -1,19 +1,19 @@
 ---
-source: parser/src/string.rs
+source: compiler/parser/src/string.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
             row: 1,
-            column: 3,
+            column: 1,
         },
         custom: (),
         node: Expr {
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 3,
+                    column: 1,
                 },
                 custom: (),
                 node: JoinedStr {
@@ -21,7 +21,7 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 3,
+                                column: 1,
                             },
                             custom: (),
                             node: Constant {
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
index 190854c3..74bf59c3 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
@@ -1,19 +1,19 @@
 ---
-source: parser/src/string.rs
+source: compiler/parser/src/string.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
             row: 1,
-            column: 3,
+            column: 1,
         },
         custom: (),
         node: Expr {
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 3,
+                    column: 1,
                 },
                 custom: (),
                 node: Constant {
diff --git a/parser/src/string.rs b/parser/src/string.rs
index 7529d3d0..4c8ff851 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -131,4 +131,11 @@ mod tests {
         let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
+
+    #[test]
+    fn test_parse_string_triple_quotes_with_kind() {
+        let source = String::from("u'''Hello, world!'''");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
 }

From 621f58407a2a0882792880e43c2ecaee695e1c87 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Sun, 16 Oct 2022 01:58:29 +0900
Subject: [PATCH 278/459] Fix nightly clippy warnings

---
 codegen/src/compile.rs | 16 ++++++++--------
 core/src/bytecode.rs   | 16 ++--------------
 2 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index ebfd454a..bdad5f7d 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -342,7 +342,7 @@ impl Compiler {
             self.emit(Instruction::StoreGlobal(doc))
         }
 
-        if self.find_ann(statements) {
+        if Self::find_ann(statements) {
             self.emit(Instruction::SetupAnnotation);
         }
 
@@ -1229,22 +1229,22 @@ impl Compiler {
     }
 
     // Python/compile.c find_ann
-    fn find_ann(&self, body: &[ast::Stmt]) -> bool {
+    fn find_ann(body: &[ast::Stmt]) -> bool {
         use ast::StmtKind::*;
 
         for statement in body {
             let res = match &statement.node {
                 AnnAssign { .. } => true,
-                For { body, orelse, .. } => self.find_ann(body) || self.find_ann(orelse),
-                If { body, orelse, .. } => self.find_ann(body) || self.find_ann(orelse),
-                While { body, orelse, .. } => self.find_ann(body) || self.find_ann(orelse),
-                With { body, .. } => self.find_ann(body),
+                For { body, orelse, .. } => Self::find_ann(body) || Self::find_ann(orelse),
+                If { body, orelse, .. } => Self::find_ann(body) || Self::find_ann(orelse),
+                While { body, orelse, .. } => Self::find_ann(body) || Self::find_ann(orelse),
+                With { body, .. } => Self::find_ann(body),
                 Try {
                     body,
                     orelse,
                     finalbody,
                     ..
-                } => self.find_ann(body) || self.find_ann(orelse) || self.find_ann(finalbody),
+                } => Self::find_ann(body) || Self::find_ann(orelse) || Self::find_ann(finalbody),
                 _ => false,
             };
             if res {
@@ -1295,7 +1295,7 @@ impl Compiler {
         let doc = self.name("__doc__");
         self.emit(Instruction::StoreLocal(doc));
         // setup annotations
-        if self.find_ann(body) {
+        if Self::find_ann(body) {
             self.emit(Instruction::SetupAnnotation);
         }
         self.compile_statements(body)?;
diff --git a/core/src/bytecode.rs b/core/src/bytecode.rs
index ae28f6ef..9c3751cd 100644
--- a/core/src/bytecode.rs
+++ b/core/src/bytecode.rs
@@ -996,20 +996,8 @@ impl Instruction {
             | SetupFinally { .. }
             | EnterFinally
             | EndFinally => 0,
-            SetupExcept { .. } => {
-                if jump {
-                    1
-                } else {
-                    0
-                }
-            }
-            SetupWith { .. } => {
-                if jump {
-                    0
-                } else {
-                    1
-                }
-            }
+            SetupExcept { .. } => jump as i32,
+            SetupWith { .. } => (!jump) as i32,
             WithCleanupStart => 0,
             WithCleanupFinish => -1,
             PopBlock => 0,

From 586e0425c14db0081082e64940b07afcb49a3fb7 Mon Sep 17 00:00:00 2001
From: Gyubong <jopemachine@naver.com>
Date: Sun, 16 Oct 2022 15:34:44 +0900
Subject: [PATCH 279/459] Fix #3934

---
 codegen/src/compile.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index ebfd454a..555eb1ad 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -2118,15 +2118,16 @@ impl Compiler {
             Name { id, .. } => self.load_name(id)?,
             Lambda { args, body } => {
                 let prev_ctx = self.ctx;
+
+                let name = "<lambda>".to_owned();
+                let mut funcflags = self.enter_function(&name, args)?;
+
                 self.ctx = CompileContext {
                     loop_data: Option::None,
                     in_class: prev_ctx.in_class,
                     func: FunctionContext::Function,
                 };
 
-                let name = "<lambda>".to_owned();
-                let mut funcflags = self.enter_function(&name, args)?;
-
                 self.current_codeinfo()
                     .constants
                     .insert_full(ConstantData::None);

From 9a1d0e9a41e5cfbda0ecaa182c7b37f7002927f3 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Sat, 8 Oct 2022 22:00:13 -0400
Subject: [PATCH 280/459] Make AST nodes Clone-able

---
 ast/asdl_rs.py      |  2 +-
 ast/src/ast_gen.rs  | 38 +++++++++++++++++++-------------------
 ast/src/constant.rs |  2 +-
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py
index 71c90328..b24ef01f 100755
--- a/ast/asdl_rs.py
+++ b/ast/asdl_rs.py
@@ -190,7 +190,7 @@ def visitSum(self, sum, name, depth):
             self.sum_with_constructors(sum, name, depth)
 
     def emit_attrs(self, depth):
-        self.emit("#[derive(Debug, PartialEq)]", depth)
+        self.emit("#[derive(Clone, Debug, PartialEq)]", depth)
 
     def simple_sum(self, sum, name, depth):
         rustname = get_rust_type(name)
diff --git a/ast/src/ast_gen.rs b/ast/src/ast_gen.rs
index 17900fe3..4460ee84 100644
--- a/ast/src/ast_gen.rs
+++ b/ast/src/ast_gen.rs
@@ -7,7 +7,7 @@ pub use crate::Location;
 
 type Ident = String;
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct Located<T, U = ()> {
     pub location: Location,
     pub custom: U,
@@ -24,7 +24,7 @@ impl<T> Located<T> {
     }
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum Mod<U = ()> {
     Module {
         body: Vec<Stmt<U>>,
@@ -42,7 +42,7 @@ pub enum Mod<U = ()> {
     },
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum StmtKind<U = ()> {
     FunctionDef {
         name: Ident,
@@ -164,7 +164,7 @@ pub enum StmtKind<U = ()> {
 }
 pub type Stmt<U = ()> = Located<StmtKind<U>, U>;
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum ExprKind<U = ()> {
     BoolOp {
         op: Boolop,
@@ -281,20 +281,20 @@ pub enum ExprKind<U = ()> {
 }
 pub type Expr<U = ()> = Located<ExprKind<U>, U>;
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum ExprContext {
     Load,
     Store,
     Del,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum Boolop {
     And,
     Or,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum Operator {
     Add,
     Sub,
@@ -311,7 +311,7 @@ pub enum Operator {
     FloorDiv,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum Unaryop {
     Invert,
     Not,
@@ -319,7 +319,7 @@ pub enum Unaryop {
     USub,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum Cmpop {
     Eq,
     NotEq,
@@ -333,7 +333,7 @@ pub enum Cmpop {
     NotIn,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct Comprehension<U = ()> {
     pub target: Box<Expr<U>>,
     pub iter: Box<Expr<U>>,
@@ -341,7 +341,7 @@ pub struct Comprehension<U = ()> {
     pub is_async: usize,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum ExcepthandlerKind<U = ()> {
     ExceptHandler {
         type_: Option<Box<Expr<U>>>,
@@ -351,7 +351,7 @@ pub enum ExcepthandlerKind<U = ()> {
 }
 pub type Excepthandler<U = ()> = Located<ExcepthandlerKind<U>, U>;
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct Arguments<U = ()> {
     pub posonlyargs: Vec<Arg<U>>,
     pub args: Vec<Arg<U>>,
@@ -362,7 +362,7 @@ pub struct Arguments<U = ()> {
     pub defaults: Vec<Expr<U>>,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct ArgData<U = ()> {
     pub arg: Ident,
     pub annotation: Option<Box<Expr<U>>>,
@@ -370,34 +370,34 @@ pub struct ArgData<U = ()> {
 }
 pub type Arg<U = ()> = Located<ArgData<U>, U>;
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct KeywordData<U = ()> {
     pub arg: Option<Ident>,
     pub value: Box<Expr<U>>,
 }
 pub type Keyword<U = ()> = Located<KeywordData<U>, U>;
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct AliasData {
     pub name: Ident,
     pub asname: Option<Ident>,
 }
 pub type Alias<U = ()> = Located<AliasData, U>;
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct Withitem<U = ()> {
     pub context_expr: Box<Expr<U>>,
     pub optional_vars: Option<Box<Expr<U>>>,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct MatchCase<U = ()> {
     pub pattern: Box<Pattern<U>>,
     pub guard: Option<Box<Expr<U>>>,
     pub body: Vec<Stmt<U>>,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum PatternKind<U = ()> {
     MatchValue {
         value: Box<Expr<U>>,
@@ -432,7 +432,7 @@ pub enum PatternKind<U = ()> {
 }
 pub type Pattern<U = ()> = Located<PatternKind<U>, U>;
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum TypeIgnore {
     TypeIgnore { lineno: usize, tag: String },
 }
diff --git a/ast/src/constant.rs b/ast/src/constant.rs
index b73bab62..5c5e55b2 100644
--- a/ast/src/constant.rs
+++ b/ast/src/constant.rs
@@ -1,7 +1,7 @@
 use num_bigint::BigInt;
 pub use rustpython_compiler_core::ConversionFlag;
 
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub enum Constant {
     None,
     Bool(bool),

From ca62bd159389d33c0f6a9407cff3e149ea528a87 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Sun, 16 Oct 2022 11:25:46 -0400
Subject: [PATCH 281/459] Start simple string at quote mark

---
 parser/src/lexer.rs                                       | 2 +-
 .../rustpython_parser__parser__tests__parse_class.snap    | 6 +++---
 .../rustpython_parser__parser__tests__parse_kwargs.snap   | 5 ++---
 .../rustpython_parser__parser__tests__parse_print_2.snap  | 5 ++---
 ...stpython_parser__parser__tests__parse_print_hello.snap | 4 ++--
 .../rustpython_parser__parser__tests__parse_string.snap   | 6 +++---
 ...on_parser__string__tests__parse_f_string_concat_1.snap | 8 ++++----
 ...on_parser__string__tests__parse_f_string_concat_2.snap | 8 ++++----
 ...on_parser__string__tests__parse_f_string_concat_3.snap | 8 ++++----
 ...python_parser__string__tests__parse_string_concat.snap | 6 +++---
 ...on_parser__string__tests__parse_u_string_concat_1.snap | 6 +++---
 11 files changed, 31 insertions(+), 33 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 5e5264fe..2f72cf78 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -492,9 +492,9 @@ where
         is_unicode: bool,
         is_fstring: bool,
     ) -> LexResult {
+        let start_pos = self.get_pos();
         let quote_char = self.next_char().unwrap();
         let mut string_content = String::new();
-        let start_pos = self.get_pos();
 
         // If the next two characters are also the quote character, then we have a triple-quoted
         // string; consume those two characters and ensure that we require a triple-quote to close
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
index 869669e9..327da516 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
@@ -1,6 +1,6 @@
 ---
-source: parser/src/parser.rs
-expression: parse_program(&source).unwrap()
+source: compiler/parser/src/parser.rs
+expression: "parse_program(source, \"<test>\").unwrap()"
 ---
 [
     Located {
@@ -126,7 +126,7 @@ expression: parse_program(&source).unwrap()
                                 Located {
                                     location: Location {
                                         row: 4,
-                                        column: 37,
+                                        column: 36,
                                     },
                                     custom: (),
                                     node: Constant {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap
index d31a3921..1b8a5451 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap
@@ -1,7 +1,6 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
-
 ---
 [
     Located {
@@ -33,7 +32,7 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 10,
+                                column: 9,
                             },
                             custom: (),
                             node: Constant {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap
index 6fe4cad3..3f828ded 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap
@@ -1,7 +1,6 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
-
 ---
 [
     Located {
@@ -33,7 +32,7 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 8,
+                                column: 7,
                             },
                             custom: (),
                             node: Constant {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap
index b4e78d8d..aae345a0 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
 ---
 [
@@ -32,7 +32,7 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 8,
+                                column: 7,
                             },
                             custom: (),
                             node: Constant {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap
index b808f28e..efbf32fd 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap
@@ -1,19 +1,19 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
             row: 1,
-            column: 2,
+            column: 1,
         },
         custom: (),
         node: Expr {
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 custom: (),
                 node: Constant {
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
index 687ee960..a486e11a 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
@@ -1,19 +1,19 @@
 ---
-source: parser/src/string.rs
+source: compiler/parser/src/string.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
             row: 1,
-            column: 2,
+            column: 1,
         },
         custom: (),
         node: Expr {
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 custom: (),
                 node: JoinedStr {
@@ -21,7 +21,7 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 2,
+                                column: 1,
                             },
                             custom: (),
                             node: Constant {
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
index 687ee960..a486e11a 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
@@ -1,19 +1,19 @@
 ---
-source: parser/src/string.rs
+source: compiler/parser/src/string.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
             row: 1,
-            column: 2,
+            column: 1,
         },
         custom: (),
         node: Expr {
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 custom: (),
                 node: JoinedStr {
@@ -21,7 +21,7 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 2,
+                                column: 1,
                             },
                             custom: (),
                             node: Constant {
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
index a41e7658..ccddd2ad 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
@@ -6,14 +6,14 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 2,
+            column: 1,
         },
         custom: (),
         node: Expr {
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 custom: (),
                 node: JoinedStr {
@@ -21,7 +21,7 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 2,
+                                column: 1,
                             },
                             custom: (),
                             node: Constant {
@@ -41,7 +41,7 @@ expression: parse_ast
                                 value: Located {
                                     location: Location {
                                         row: 1,
-                                        column: 3,
+                                        column: 2,
                                     },
                                     custom: (),
                                     node: Constant {
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
index 25ed2337..380a1fc8 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
@@ -1,19 +1,19 @@
 ---
-source: parser/src/string.rs
+source: compiler/parser/src/string.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
             row: 1,
-            column: 2,
+            column: 1,
         },
         custom: (),
         node: Expr {
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 custom: (),
                 node: Constant {
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
index 25ed2337..380a1fc8 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
@@ -1,19 +1,19 @@
 ---
-source: parser/src/string.rs
+source: compiler/parser/src/string.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
             row: 1,
-            column: 2,
+            column: 1,
         },
         custom: (),
         node: Expr {
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 custom: (),
                 node: Constant {

From 8a32bab00a196039fb517a9dc346e53084ae4a2a Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Mon, 17 Oct 2022 00:18:30 -0400
Subject: [PATCH 282/459] Add end locations to all nodes (#4192)

---
 ast/asdl_rs.py                                |  10 +-
 ast/src/ast_gen.rs                            |   5 +-
 ast/src/constant.rs                           |  28 +-
 parser/python.lalrpop                         | 249 +++++++++++-------
 parser/src/fstring.rs                         |  24 +-
 parser/src/function.rs                        |  12 +-
 parser/src/parser.rs                          |   3 +
 ...rser__context__tests__ann_assign_name.snap |  69 +++++
 ...ser__context__tests__assign_attribute.snap | 119 +++++++++
 ...on_parser__context__tests__assign_for.snap | 117 ++++++++
 ...n_parser__context__tests__assign_list.snap | 135 ++++++++++
 ...ser__context__tests__assign_list_comp.snap | 153 +++++++++++
 ...n_parser__context__tests__assign_name.snap | 104 ++++++++
 ...er__context__tests__assign_named_expr.snap |  79 ++++++
 ...rser__context__tests__assign_set_comp.snap | 153 +++++++++++
 ...arser__context__tests__assign_starred.snap | 149 +++++++++++
 ...ser__context__tests__assign_subscript.snap | 133 ++++++++++
 ..._parser__context__tests__assign_tuple.snap | 135 ++++++++++
 ...n_parser__context__tests__assign_with.snap |  72 +++++
 ..._context__tests__aug_assign_attribute.snap | 117 ++++++++
 ...rser__context__tests__aug_assign_name.snap |  52 ++++
 ..._context__tests__aug_assign_subscript.snap | 131 +++++++++
 ...parser__context__tests__del_attribute.snap |  51 ++++
 ...thon_parser__context__tests__del_name.snap |  36 +++
 ...parser__context__tests__del_subscript.snap |  65 +++++
 ...s__fstring_parse_selfdocumenting_base.snap |  24 ++
 ...tring_parse_selfdocumenting_base_more.snap |  60 +++++
 ..._fstring_parse_selfdocumenting_format.snap |  36 +++
 ...parser__fstring__tests__parse_fstring.snap |  30 +++
 ..._fstring__tests__parse_fstring_equals.snap |  24 ++
 ...ing__tests__parse_fstring_nested_spec.snap |  30 +++
 ...ring__tests__parse_fstring_not_equals.snap |  24 ++
 ..._tests__parse_fstring_not_nested_spec.snap |  24 ++
 ...sts__parse_fstring_selfdoc_prec_space.snap |  24 ++
 ..._parse_fstring_selfdoc_trailing_space.snap |  24 ++
 ...ring__tests__parse_fstring_yield_expr.snap |  12 +
 ...on_parser__parser__tests__parse_class.snap |  66 +++++
 ...rser__tests__parse_dict_comprehension.snap |  32 ++-
 ...ests__parse_double_list_comprehension.snap |  86 +++++-
 ...parser__parser__tests__parse_f_string.snap |  18 ++
 ..._tests__parse_generator_comprehension.snap |  26 +-
 ...er__parser__tests__parse_if_elif_else.snap |  63 ++++-
 ...parse_if_else_generator_comprehension.snap |  44 +++-
 ...n_parser__parser__tests__parse_kwargs.snap |  38 ++-
 ...n_parser__parser__tests__parse_lambda.snap |  44 +++-
 ...rser__tests__parse_list_comprehension.snap |  26 +-
 ...ed_expression_generator_comprehension.snap |  50 +++-
 ..._parser__parser__tests__parse_print_2.snap |  32 ++-
 ...ser__parser__tests__parse_print_hello.snap |  26 +-
 ...n_parser__parser__tests__parse_string.snap |  12 +
 ...n_parser__parser__tests__parse_tuples.snap |  47 +++-
 ...tring__tests__parse_f_string_concat_1.snap |  18 ++
 ...tring__tests__parse_f_string_concat_2.snap |  18 ++
 ...tring__tests__parse_f_string_concat_3.snap |  30 +++
 ...r__string__tests__parse_string_concat.snap |  12 +
 ..._parse_string_triple_quotes_with_kind.snap |  12 +
 ...ing__tests__parse_u_f_string_concat_1.snap |  18 ++
 ...ing__tests__parse_u_f_string_concat_2.snap |  18 ++
 ...tring__tests__parse_u_string_concat_1.snap |  12 +
 ...tring__tests__parse_u_string_concat_2.snap |  12 +
 parser/src/string.rs                          |  28 +-
 61 files changed, 3157 insertions(+), 144 deletions(-)
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__ann_assign_name.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__assign_with.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__aug_assign_name.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__del_attribute.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__del_name.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__del_subscript.snap

diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py
index 71c90328..ea035ede 100755
--- a/ast/asdl_rs.py
+++ b/ast/asdl_rs.py
@@ -304,7 +304,7 @@ def visitModule(self, mod, depth):
             depth,
         )
         self.emit(
-            "Ok(Located { custom: folder.map_user(node.custom)?, location: node.location, node: f(folder, node.node)? })",
+            "Ok(Located { custom: folder.map_user(node.custom)?, location: node.location, end_location: node.end_location, node: f(folder, node.node)? })",
             depth + 1,
         )
         self.emit("}", depth)
@@ -617,9 +617,6 @@ def extract_location(self, typename, depth):
         column = self.decode_field(asdl.Field("int", "col_offset"), typename)
         self.emit(f"let _location = ast::Location::new({row}, {column});", depth)
 
-    def wrap_located_node(self, depth):
-        self.emit(f"let node = ast::Located::new(_location, node);", depth)
-
     def decode_field(self, field, typename):
         name = json.dumps(field.name)
         if field.opt and not field.seq:
@@ -658,13 +655,14 @@ def write_ast_def(mod, typeinfo, f):
             """
         pub struct Located<T, U = ()> {
             pub location: Location,
+            pub end_location: Option<Location>,
             pub custom: U,
             pub node: T,
         }
     
         impl<T> Located<T> {
-            pub fn new(location: Location, node: T) -> Self {
-                Self { location, custom: (), node }
+            pub fn new(location: Location, end_location: Location, node: T) -> Self {
+                Self { location, end_location: Some(end_location), custom: (), node }
             }
         }
         \n
diff --git a/ast/src/ast_gen.rs b/ast/src/ast_gen.rs
index 17900fe3..e4fcc44b 100644
--- a/ast/src/ast_gen.rs
+++ b/ast/src/ast_gen.rs
@@ -10,14 +10,16 @@ type Ident = String;
 #[derive(Debug, PartialEq)]
 pub struct Located<T, U = ()> {
     pub location: Location,
+    pub end_location: Option<Location>,
     pub custom: U,
     pub node: T,
 }
 
 impl<T> Located<T> {
-    pub fn new(location: Location, node: T) -> Self {
+    pub fn new(location: Location, end_location: Location, node: T) -> Self {
         Self {
             location,
+            end_location: Some(end_location),
             custom: (),
             node,
         }
@@ -529,6 +531,7 @@ pub mod fold {
         Ok(Located {
             custom: folder.map_user(node.custom)?,
             location: node.location,
+            end_location: node.end_location,
             node: f(folder, node.node)?,
         })
     }
diff --git a/ast/src/constant.rs b/ast/src/constant.rs
index b73bab62..5de3b346 100644
--- a/ast/src/constant.rs
+++ b/ast/src/constant.rs
@@ -121,6 +121,7 @@ impl<U> crate::fold::Fold<U> for ConstantOptimizer {
                     node: expr,
                     custom: node.custom,
                     location: node.location,
+                    end_location: node.end_location,
                 })
             }
             _ => crate::fold::fold_expr(self, node),
@@ -137,16 +138,19 @@ mod tests {
         use crate::fold::Fold;
         use crate::*;
 
-        let location = Location::new(0, 0);
+        let start = Default::default();
+        let end = None;
         let custom = ();
         let ast = Located {
-            location,
+            location: start,
+            end_location: end,
             custom,
             node: ExprKind::Tuple {
                 ctx: ExprContext::Load,
                 elts: vec![
                     Located {
-                        location,
+                        location: start,
+                        end_location: end,
                         custom,
                         node: ExprKind::Constant {
                             value: BigInt::from(1).into(),
@@ -154,7 +158,8 @@ mod tests {
                         },
                     },
                     Located {
-                        location,
+                        location: start,
+                        end_location: end,
                         custom,
                         node: ExprKind::Constant {
                             value: BigInt::from(2).into(),
@@ -162,13 +167,15 @@ mod tests {
                         },
                     },
                     Located {
-                        location,
+                        location: start,
+                        end_location: end,
                         custom,
                         node: ExprKind::Tuple {
                             ctx: ExprContext::Load,
                             elts: vec![
                                 Located {
-                                    location,
+                                    location: start,
+                                    end_location: end,
                                     custom,
                                     node: ExprKind::Constant {
                                         value: BigInt::from(3).into(),
@@ -176,7 +183,8 @@ mod tests {
                                     },
                                 },
                                 Located {
-                                    location,
+                                    location: start,
+                                    end_location: end,
                                     custom,
                                     node: ExprKind::Constant {
                                         value: BigInt::from(4).into(),
@@ -184,7 +192,8 @@ mod tests {
                                     },
                                 },
                                 Located {
-                                    location,
+                                    location: start,
+                                    end_location: end,
                                     custom,
                                     node: ExprKind::Constant {
                                         value: BigInt::from(5).into(),
@@ -203,7 +212,8 @@ mod tests {
         assert_eq!(
             new_ast,
             Located {
-                location,
+                location: start,
+                end_location: end,
                 custom,
                 node: ExprKind::Constant {
                     value: Constant::Tuple(vec![
diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 224450e0..cf43f0f2 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -66,9 +66,10 @@ SmallStatement: ast::Stmt = {
 };
 
 PassStatement: ast::Stmt = {
-    <location:@L> "pass" => {
+    <location:@L> "pass" <end_location:@R> => {
         ast::Stmt {
             location,
+            end_location: Some(end_location),
             custom: (),
             node: ast::StmtKind::Pass,
         }
@@ -76,9 +77,10 @@ PassStatement: ast::Stmt = {
 };
 
 DelStatement: ast::Stmt = {
-    <location:@L> "del" <targets:ExpressionList2> => {
+    <location:@L> "del" <targets:ExpressionList2> <end_location:@R> => {
         ast::Stmt {
             location,
+            end_location: Some(end_location),
             custom: (),
             node: ast::StmtKind::Delete { targets },
         }
@@ -86,12 +88,13 @@ DelStatement: ast::Stmt = {
 };
 
 ExpressionStatement: ast::Stmt = {
-    <location:@L> <expression:TestOrStarExprList> <suffix:AssignSuffix*> => {
+    <location:@L> <expression:TestOrStarExprList> <suffix:AssignSuffix*> <end_location:@R> => {
         // Just an expression, no assignment:
         if suffix.is_empty() {
             ast::Stmt {
                 custom: (),
                 location,
+                end_location: Some(end_location),
                 node: ast::StmtKind::Expr { value: Box::new(expression) }
             }
         } else {
@@ -107,14 +110,16 @@ ExpressionStatement: ast::Stmt = {
             ast::Stmt {
                 custom: (),
                 location,
+                end_location: Some(end_location),
                 node: ast::StmtKind::Assign { targets, value, type_comment: None },
             }
         }
     },
-    <location:@L> <target:TestOrStarExprList> <op:AugAssign> <rhs:TestListOrYieldExpr> => {
+    <location:@L> <target:TestOrStarExprList> <op:AugAssign> <rhs:TestListOrYieldExpr> <end_location:@R> => {
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::AugAssign {
                 target: Box::new(target),
                 op,
@@ -122,11 +127,12 @@ ExpressionStatement: ast::Stmt = {
             },
         }
     },
-    <location:@L> <target:Test> ":" <annotation:Test> <rhs:AssignSuffix?> => {
+    <location:@L> <target:Test> ":" <annotation:Test> <rhs:AssignSuffix?> <end_location:@R> => {
         let simple = matches!(target.node, ast::ExprKind::Name { .. });
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::AnnAssign {
                 target: Box::new(target),
                 annotation: Box::new(annotation),
@@ -183,31 +189,35 @@ AugAssign: ast::Operator = {
 };
 
 FlowStatement: ast::Stmt = {
-    <location:@L> "break" => {
+    <location:@L> "break" <end_location:@R> => {
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::Break,
         }
     },
-    <location:@L> "continue" => {
+    <location:@L> "continue" <end_location:@R> => {
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::Continue,
         }
     },
-    <location:@L> "return" <value:TestList?> => {
+    <location:@L> "return" <value:TestList?> <end_location:@R> => {
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::Return { value: value.map(Box::new) },
         }
     },
-    <location:@L> <expression:YieldExpr> => {
+    <location:@L> <expression:YieldExpr> <end_location:@R> => {
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::Expr { value: Box::new(expression) },
         }
     },
@@ -215,35 +225,39 @@ FlowStatement: ast::Stmt = {
 };
 
 RaiseStatement: ast::Stmt = {
-    <location:@L> "raise" => {
+    <location:@L> "raise" <end_location:@R> => {
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::Raise { exc: None, cause: None },
         }
     },
-    <location:@L> "raise" <t:Test> <c:("from" Test)?> => {
+    <location:@L> "raise" <t:Test> <c:("from" Test)?> <end_location:@R> => {
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::Raise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)) },
         }
     },
 };
 
 ImportStatement: ast::Stmt = {
-    <location:@L> "import" <names: OneOrMore<ImportAsAlias<DottedName>>> => {
+    <location:@L> "import" <names: OneOrMore<ImportAsAlias<DottedName>>> <end_location:@R> => {
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::Import { names },
         }
     },
-    <location:@L> "from" <source:ImportFromLocation> "import" <names: ImportAsNames> => {
+    <location:@L> "from" <source:ImportFromLocation> "import" <names: ImportAsNames> <end_location:@R> => {
         let (level, module) = source;
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::ImportFrom {
                 level,
                 module,
@@ -268,18 +282,18 @@ ImportDots: usize = {
 };
 
 ImportAsNames: Vec<ast::Alias> = {
-    <location:@L> <i:OneOrMore<ImportAsAlias<Identifier>>> => i,
-    <location:@L> "(" <i:OneOrMore<ImportAsAlias<Identifier>>> ","? ")" => i,
-    <location:@L> "*" => {
+    <location:@L> <i:OneOrMore<ImportAsAlias<Identifier>>> <end_location:@R> => i,
+    <location:@L> "(" <i:OneOrMore<ImportAsAlias<Identifier>>> ","? ")" <end_location:@R> => i,
+    <location:@L> "*" <end_location:@R> => {
         // Star import all
-        vec![ast::Alias::new(location, ast::AliasData { name: "*".to_string(), asname: None })]
+        vec![ast::Alias::new(location, end_location, ast::AliasData { name: "*".to_string(), asname: None })]
     },
 };
 
 
 #[inline]
 ImportAsAlias<I>: ast::Alias = {
-    <location:@L> <name:I> <a: ("as" Identifier)?> => ast::Alias::new(location, ast::AliasData { name, asname: a.map(|a| a.1) }),
+    <location:@L> <name:I> <a: ("as" Identifier)?> <end_location:@R> => ast::Alias::new(location, end_location, ast::AliasData { name, asname: a.map(|a| a.1) }),
 }
 
 // A name like abc or abc.def.ghi
@@ -296,30 +310,33 @@ DottedName: String = {
 };
 
 GlobalStatement: ast::Stmt = {
-    <location:@L> "global" <names:OneOrMore<Identifier>> => {
+    <location:@L> "global" <names:OneOrMore<Identifier>> <end_location:@R> => {
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::Global { names }
         }
     },
 };
 
 NonlocalStatement: ast::Stmt = {
-    <location:@L> "nonlocal" <names:OneOrMore<Identifier>> => {
+    <location:@L> "nonlocal" <names:OneOrMore<Identifier>> <end_location:@R> => {
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::Nonlocal { names }
         }
     },
 };
 
 AssertStatement: ast::Stmt = {
-    <location:@L> "assert" <test:Test> <msg: ("," Test)?> => {
+    <location:@L> "assert" <test:Test> <msg: ("," Test)?> <end_location:@R> => {
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::Assert {
                 test: Box::new(test),
                 msg: msg.map(|e| Box::new(e.1))
@@ -339,7 +356,7 @@ CompoundStatement: ast::Stmt = {
 };
 
 IfStatement: ast::Stmt = {
-    <location:@L> "if" <test:NamedExpressionTest> ":" <body:Suite> <s2:(@L "elif" NamedExpressionTest ":" Suite)*> <s3:("else" ":" Suite)?> => {
+    <location:@L> "if" <test:NamedExpressionTest> ":" <body:Suite> <s2:(@L "elif" NamedExpressionTest ":" Suite @R)*> <s3:("else" ":" Suite)?> <end_location:@R> => {
         // Determine last else:
         let mut last = s3.map(|s| s.2).unwrap_or_default();
 
@@ -348,6 +365,7 @@ IfStatement: ast::Stmt = {
             let x = ast::Stmt {
                 custom: (),
                 location: i.0,
+                end_location: Some(i.5),
                 node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last },
             };
             last = vec![x];
@@ -356,17 +374,19 @@ IfStatement: ast::Stmt = {
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::If { test: Box::new(test), body, orelse: last }
         }
     },
 };
 
 WhileStatement: ast::Stmt = {
-    <location:@L> "while" <test:NamedExpressionTest> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
+    <location:@L> "while" <test:NamedExpressionTest> ":" <body:Suite> <s2:("else" ":" Suite)?> <end_location:@R> => {
         let orelse = s2.map(|s| s.2).unwrap_or_default();
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::While {
                 test: Box::new(test),
                 body,
@@ -377,7 +397,7 @@ WhileStatement: ast::Stmt = {
 };
 
 ForStatement: ast::Stmt = {
-    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
+    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> <end_location:@R> => {
         let orelse = s2.map(|s| s.2).unwrap_or_default();
         let target = Box::new(target);
         let iter = Box::new(iter);
@@ -387,17 +407,18 @@ ForStatement: ast::Stmt = {
         } else {
             ast::StmtKind::For { target, iter, body, orelse, type_comment }
         };
-        ast::Stmt::new(location, node)
+        ast::Stmt::new(location, end_location, node)
     },
 };
 
 TryStatement: ast::Stmt = {
-    <location:@L> "try" ":" <body:Suite> <handlers:ExceptClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> => {
+    <location:@L> "try" ":" <body:Suite> <handlers:ExceptClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> <end_location:@R> => {
         let orelse = else_suite.map(|s| s.2).unwrap_or_default();
         let finalbody = finally.map(|s| s.2).unwrap_or_default();
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::Try {
                 body,
                 handlers,
@@ -406,13 +427,14 @@ TryStatement: ast::Stmt = {
             },
         }
     },
-    <location:@L> "try" ":" <body:Suite> <finally:("finally" ":" Suite)> => {
+    <location:@L> "try" ":" <body:Suite> <finally:("finally" ":" Suite)> <end_location:@R> => {
         let handlers = vec![];
         let orelse = vec![];
         let finalbody = finally.2;
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::Try {
                 body,
                 handlers,
@@ -424,9 +446,10 @@ TryStatement: ast::Stmt = {
 };
 
 ExceptClause: ast::Excepthandler = {
-    <location:@L> "except" <typ:Test?> ":" <body:Suite> => {
+    <location:@L> "except" <typ:Test?> ":" <body:Suite> <end_location:@R> => {
         ast::Excepthandler::new(
             location,
+            end_location,
             ast::ExcepthandlerKind::ExceptHandler {
                 type_: typ.map(Box::new),
                 name: None,
@@ -434,9 +457,10 @@ ExceptClause: ast::Excepthandler = {
             },
         )
     },
-    <location:@L> "except" <x:(Test "as" Identifier)> ":" <body:Suite> => {
+    <location:@L> "except" <x:(Test "as" Identifier)> ":" <body:Suite> <end_location:@R> => {
         ast::Excepthandler::new(
             location,
+            end_location,
             ast::ExcepthandlerKind::ExceptHandler {
                 type_: Some(Box::new(x.0)),
                 name: Some(x.2),
@@ -447,14 +471,14 @@ ExceptClause: ast::Excepthandler = {
 };
 
 WithStatement: ast::Stmt = {
-    <location:@L> <is_async:"async"?> "with" <items:OneOrMore<WithItem>> ":" <body:Suite> => {
+    <location:@L> <is_async:"async"?> "with" <items:OneOrMore<WithItem>> ":" <body:Suite> <end_location:@R> => {
         let type_comment = None;
         let node = if is_async.is_some() {
             ast::StmtKind::AsyncWith { items, body, type_comment }
         } else {
             ast::StmtKind::With { items, body, type_comment }
         };
-        ast::Stmt::new(location, node)
+        ast::Stmt::new(location, end_location, node)
     },
 };
 
@@ -467,7 +491,7 @@ WithItem: ast::Withitem = {
 };
 
 FuncDef: ast::Stmt = {
-    <decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test)?> ":" <body:Suite> => {
+    <decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test)?> ":" <body:Suite> <end_location:@R>  => {
         let args = Box::new(args);
         let returns = r.map(|x| Box::new(x.1));
         let type_comment = None;
@@ -476,7 +500,7 @@ FuncDef: ast::Stmt = {
         } else {
             ast::StmtKind::FunctionDef { name, args, body, decorator_list, returns, type_comment }
         };
-        ast::Stmt::new(location, node)
+        ast::Stmt::new(location, end_location, node)
     },
 };
 
@@ -574,16 +598,17 @@ ParameterDef<ArgType>: (ast::Arg, Option<ast::Expr>) = {
 };
 
 UntypedParameter: ast::Arg = {
-    <location:@L> <arg:Identifier> => ast::Arg::new(
+    <location:@L> <arg:Identifier> <end_location:@R> => ast::Arg::new(
         location,
+        end_location,
         ast::ArgData { arg, annotation: None, type_comment: None },
     ),
 };
 
 TypedParameter: ast::Arg = {
-    <location:@L> <arg:Identifier> <a:(":" Test)?>=> {
+    <location:@L> <arg:Identifier> <a:(":" Test)?> <end_location:@R> => {
         let annotation = a.map(|x| Box::new(x.1));
-        ast::Arg::new(location, ast::ArgData { arg, annotation, type_comment: None })
+        ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None })
     },
 };
 
@@ -620,7 +645,7 @@ KwargParameter<ArgType>: Option<Box<ast::Arg>> = {
 };
 
 ClassDef: ast::Stmt = {
-    <decorator_list:Decorator*> <location:@L> "class" <name:Identifier> <a:("(" ArgumentList ")")?> ":" <body:Suite> => {
+    <decorator_list:Decorator*> <location:@L> "class" <name:Identifier> <a:("(" ArgumentList ")")?> ":" <body:Suite> <end_location:@R> => {
         let (bases, keywords) = match a {
             Some((_, arg, _)) => (arg.args, arg.keywords),
             None => (vec![], vec![]),
@@ -628,6 +653,7 @@ ClassDef: ast::Stmt = {
         ast::Stmt {
             custom: (),
             location,
+            end_location: Some(end_location),
             node: ast::StmtKind::ClassDef {
                 name,
                 bases,
@@ -641,29 +667,32 @@ ClassDef: ast::Stmt = {
 
 // Decorators:
 Decorator: ast::Expr = {
-    <location:@L>"@" <p:NamedExpressionTest> "\n" => {
+    <location:@L> "@" <p:NamedExpressionTest> "\n" => {
         p
     },
 };
 
 YieldExpr: ast::Expr = {
-    <location:@L> "yield" <value:TestList?> => ast::Expr {
+    <location:@L> "yield" <value:TestList?> <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Yield { value: value.map(Box::new) }
     },
-    <location:@L> "yield" "from" <e:Test> => ast::Expr {
+    <location:@L> "yield" "from" <e:Test> <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::YieldFrom { value: Box::new(e) }
     },
 };
 
 Test: ast::Expr = {
-    <expr:OrTest> <condition: (@L "if" OrTest "else" Test)?> => {
+    <expr:OrTest> <condition: (@L "if" OrTest "else" Test @R)?> => {
         if let Some(c) = condition {
             ast::Expr {
                 location: c.0,
+                end_location: Some(c.5),
                 custom: (),
                 node: ast::ExprKind::IfExp {
                     test: Box::new(c.2),
@@ -679,14 +708,16 @@ Test: ast::Expr = {
 };
 
 NamedExpressionTest: ast::Expr = {
-    <location:@L><left: (Identifier ":=")?> <right:Test> => {
+    <location:@L> <left: (Identifier ":=")?> <right:Test> <end_location:@R> => {
         if let Some(l) = left {
             ast::Expr {
-                location: location,
+                location,
+                end_location: Some(end_location),
                 custom: (),
                 node: ast::ExprKind::NamedExpr {
                     target: Box::new(ast::Expr::new(
                         location,
+                        end_location,
                         ast::ExprKind::Name { id: l.0, ctx: ast::ExprContext::Store },
                     )),
                     value: Box::new(right),
@@ -699,7 +730,7 @@ NamedExpressionTest: ast::Expr = {
 }
 
 LambdaDef: ast::Expr = {
-    <location:@L> "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test> => {
+    <location:@L> "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test> <end_location:@R> => {
         let p = p.unwrap_or_else(|| {
             ast::Arguments {
                 posonlyargs: vec![],
@@ -713,6 +744,7 @@ LambdaDef: ast::Expr = {
         });
         ast::Expr {
             location,
+            end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::Lambda {
                 args: Box::new(p),
@@ -723,7 +755,7 @@ LambdaDef: ast::Expr = {
 }
 
 OrTest: ast::Expr = {
-    <e1:AndTest> <location:@L> <e2:("or" AndTest)*> => {
+    <e1:AndTest> <location:@L> <e2:("or" AndTest)*> <end_location:@R> => {
         if e2.is_empty() {
             e1
         } else {
@@ -731,6 +763,7 @@ OrTest: ast::Expr = {
             values.extend(e2.into_iter().map(|e| e.1));
             ast::Expr {
                 location,
+                end_location: Some(end_location),
                 custom: (),
                 node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values }
             }
@@ -739,7 +772,7 @@ OrTest: ast::Expr = {
 };
 
 AndTest: ast::Expr = {
-    <e1:NotTest> <location:@L> <e2:("and" NotTest)*> => {
+    <e1:NotTest> <location:@L> <e2:("and" NotTest)*> <end_location:@R> => {
         if e2.is_empty() {
             e1
         } else {
@@ -747,6 +780,7 @@ AndTest: ast::Expr = {
             values.extend(e2.into_iter().map(|e| e.1));
             ast::Expr {
                 location,
+                end_location: Some(end_location),
                 custom: (),
                 node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values }
             }
@@ -755,8 +789,9 @@ AndTest: ast::Expr = {
 };
 
 NotTest: ast::Expr = {
-    <location:@L> "not" <e:NotTest> => ast::Expr {
+    <location:@L> "not" <e:NotTest> <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::UnaryOp { operand: Box::new(e), op: ast::Unaryop::Not }
     },
@@ -764,10 +799,11 @@ NotTest: ast::Expr = {
 };
 
 Comparison: ast::Expr = {
-    <left:Expression> <location:@L> <comparisons:(CompOp Expression)+> => {
+    <left:Expression> <location:@L> <comparisons:(CompOp Expression)+> <end_location:@R> => {
         let (ops, comparators) = comparisons.into_iter().unzip();
         ast::Expr {
             location,
+            end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::Compare { left: Box::new(left), ops, comparators }
         }
@@ -789,8 +825,9 @@ CompOp: ast::Cmpop = {
 };
 
 Expression: ast::Expr = {
-    <e1:Expression> <location:@L> "|" <e2:XorExpression> => ast::Expr {
+    <e1:Expression> <location:@L> "|" <e2:XorExpression> <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) }
     },
@@ -798,8 +835,9 @@ Expression: ast::Expr = {
 };
 
 XorExpression: ast::Expr = {
-    <e1:XorExpression> <location:@L> "^" <e2:AndExpression> => ast::Expr {
+    <e1:XorExpression> <location:@L> "^" <e2:AndExpression> <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) }
     },
@@ -807,8 +845,9 @@ XorExpression: ast::Expr = {
 };
 
 AndExpression: ast::Expr = {
-    <e1:AndExpression> <location:@L> "&" <e2:ShiftExpression> => ast::Expr {
+    <e1:AndExpression> <location:@L> "&" <e2:ShiftExpression> <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) }
     },
@@ -816,8 +855,9 @@ AndExpression: ast::Expr = {
 };
 
 ShiftExpression: ast::Expr = {
-    <e1:ShiftExpression> <location:@L> <op:ShiftOp> <e2:ArithmeticExpression> => ast::Expr {
+    <e1:ShiftExpression> <location:@L> <op:ShiftOp> <e2:ArithmeticExpression> <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) }
     },
@@ -830,8 +870,9 @@ ShiftOp: ast::Operator = {
 };
 
 ArithmeticExpression: ast::Expr = {
-    <a:ArithmeticExpression> <location:@L> <op:AddOp> <b:Term> => ast::Expr {
+    <a:ArithmeticExpression> <location:@L> <op:AddOp> <b:Term> <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
     },
@@ -844,8 +885,9 @@ AddOp: ast::Operator = {
 };
 
 Term: ast::Expr = {
-    <a:Term> <location:@L> <op:MulOp> <b:Factor> => ast::Expr {
+    <a:Term> <location:@L> <op:MulOp> <b:Factor> <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
     },
@@ -861,8 +903,9 @@ MulOp: ast::Operator = {
 };
 
 Factor: ast::Expr = {
-    <location:@L> <op:UnaryOp> <e:Factor> => ast::Expr {
+    <location:@L> <op:UnaryOp> <e:Factor> <end_location:@R>  => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::UnaryOp { operand: Box::new(e), op }
     },
@@ -876,11 +919,12 @@ UnaryOp: ast::Unaryop = {
 };
 
 Power: ast::Expr = {
-    <e:AtomExpr> <e2:(@L "**" Factor)?> => {
+    <e:AtomExpr> <e2:(@L "**" Factor @R)?> => {
         match e2 {
             None => e,
-            Some((location, _, b)) => ast::Expr {
+            Some((location, _, b, end_location)) => ast::Expr {
                 location,
+                end_location: Some(end_location),
                 custom: (),
                 node: ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) }
             },
@@ -889,10 +933,11 @@ Power: ast::Expr = {
 };
 
 AtomExpr: ast::Expr = {
-    <location:@L> <is_await:"await"?> <atom:AtomExpr2> => {
+    <location:@L> <is_await:"await"?> <atom:AtomExpr2> <end_location:@R> => {
         if is_await.is_some() {
             ast::Expr {
                 location,
+                end_location: Some(end_location),
                 custom: (),
                 node: ast::ExprKind::Await { value: Box::new(atom) }
             }
@@ -904,27 +949,30 @@ AtomExpr: ast::Expr = {
 
 AtomExpr2: ast::Expr = {
     Atom,
-    <f:AtomExpr2> <location:@L> "(" <a:ArgumentList> ")" => {
+    <location:@L> <f:AtomExpr2> "(" <a:ArgumentList> ")" <end_location:@R> => {
         ast::Expr {
             location,
+            end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::Call { func: Box::new(f), args: a.args, keywords: a.keywords }
         }
     },
-    <e:AtomExpr2> <location:@L> "[" <s:SubscriptList> "]" => ast::Expr {
+    <location:@L> <e:AtomExpr2> "[" <s:SubscriptList> "]" <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Subscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load }
     },
-    <e:AtomExpr2> <location:@L> "." <attr:Identifier> => ast::Expr {
+    <location:@L> <e:AtomExpr2> "." <attr:Identifier> <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Attribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load }
     },
 };
 
 SubscriptList: ast::Expr = {
-    <location:@L> <s1:Subscript> <s2:("," Subscript)*> <trailing_comma:","?> => {
+    <location:@L> <s1:Subscript> <s2:("," Subscript)*> <trailing_comma:","?> <end_location:@R> => {
         if s2.is_empty() && trailing_comma.is_none() {
             s1
         } else {
@@ -935,6 +983,7 @@ SubscriptList: ast::Expr = {
 
             ast::Expr {
                 location,
+                end_location: Some(end_location),
                 custom: (),
                 node: ast::ExprKind::Tuple { elts: dims, ctx: ast::ExprContext::Load },
             }
@@ -944,12 +993,13 @@ SubscriptList: ast::Expr = {
 
 Subscript: ast::Expr = {
     Test,
-    <e1:Test?> <location:@L> ":" <e2:Test?> <e3:SliceOp?> => {
+    <e1:Test?> <location:@L> ":" <e2:Test?> <e3:SliceOp?> <end_location:@R> => {
         let lower = e1.map(Box::new);
         let upper = e2.map(Box::new);
         let step = e3.flatten().map(Box::new);
         ast::Expr {
             location,
+            end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::Slice { lower, upper, step }
         }
@@ -961,40 +1011,44 @@ SliceOp: Option<ast::Expr> = {
 }
 
 Atom: ast::Expr = {
-    <location:@L> <s:(@L string)+> =>? parse_strings(s).map_err(|e| e.into()),
-    <location:@L> <value:Constant> => ast::Expr {
+    <location:@L> <s:(@L string @R)+> =>? parse_strings(s).map_err(|e| e.into()),
+    <location:@L> <value:Constant> <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Constant { value, kind: None }
     },
-    <location:@L> <name:Identifier> => ast::Expr {
+    <location:@L> <name:Identifier> <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load }
     },
-    <location:@L> "[" <e:ListLiteralValues?> "]" => {
+    <location:@L> "[" <e:ListLiteralValues?> "]"<end_location:@R>  => {
         let elts = e.unwrap_or_default();
         ast::Expr {
             location,
+            end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::List { elts, ctx: ast::ExprContext::Load }
         }
     },
-    <location:@L> "[" <elt:TestOrStarNamedExpr> <generators:CompFor> "]" => {
+    <location:@L> "[" <elt:TestOrStarNamedExpr> <generators:CompFor> "]" <end_location:@R> => {
         ast::Expr {
             location,
+            end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::ListComp { elt: Box::new(elt), generators }
         }
     },
-    <location:@L> "(" <elements:TestOrStarNamedExprList?> ")" =>? {
+    <location:@L> "(" <elements:TestOrStarNamedExprList?> ")" <end_location:@R> =>? {
         match elements {
             Some(elt) => {
                 match elt.node {
                     ast::ExprKind::Starred { .. } => {
                         Err(LexicalError{
                             error : LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
-                            location,
+                            location: location,
                         }.into())
                     },
                     _ => {
@@ -1003,29 +1057,30 @@ Atom: ast::Expr = {
                 }
             },
             None => {
-                Ok(ast::Expr {
+                Ok(ast::Expr::new(
                      location,
-                     custom: (),
-                     node: ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load }
-                })
+                     end_location,
+                     ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load }
+                ))
             }
         }
     },
     "(" <e:YieldExpr> ")" => e,
-    <location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" => {
+    <location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" <end_location:@R> => {
         ast::Expr {
             location,
+            end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::GeneratorExp { elt: Box::new(elt), generators }
         }
     },
-    "(" <location:@L> "**" <e:Expression> ")" =>? {
+    "(" <location:@L> "**" <e:Expression> ")" <end_location:@R> =>? {
         Err(LexicalError{
             error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()),
-            location,
+            location: location,
         }.into())
     },
-    <location:@L> "{" <e:DictLiteralValues?> "}" => {
+    <location:@L> "{" <e:DictLiteralValues?> "}" <end_location:@R> => {
         let pairs = e.unwrap_or_default();
 
         let (keys, values) = match pairs.iter().position(|(k,_)| k.is_none()) {
@@ -1035,9 +1090,11 @@ Atom: ast::Expr = {
 
                 fn build_map(items: &mut Vec<(ast::Expr, ast::Expr)>) -> ast::Expr {
                     let location = items[0].0.location;
+                    let end_location = items[0].0.end_location;
                     let (keys, values) = items.drain(..).unzip();
                     ast::Expr {
                         location,
+                        end_location,
                         custom: (),
                         node: ast::ExprKind::Dict { keys, values }
                     }
@@ -1064,13 +1121,15 @@ Atom: ast::Expr = {
 
         ast::Expr {
             location,
+            end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::Dict { keys, values }
         }
     },
-    <location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" => {
+    <location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" <end_location:@R> => {
         ast::Expr {
             location,
+            end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::DictComp {
                 key: Box::new(e1.0),
@@ -1079,22 +1138,24 @@ Atom: ast::Expr = {
             }
         }
     },
-    <location:@L> "{" <elts:SetLiteralValues> "}" => ast::Expr {
+    <location:@L> "{" <elts:SetLiteralValues> "}" <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Set { elts }
     },
-    <location:@L> "{" <elt:Test> <generators:CompFor> "}" => {
+    <location:@L> "{" <elt:Test> <generators:CompFor> "}" <end_location:@R> => {
         ast::Expr {
             location,
+            end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::SetComp { elt: Box::new(elt), generators }
         }
     },
-    <location:@L> "True" => ast::Expr::new(location, ast::ExprKind::Constant { value: true.into(), kind: None }),
-    <location:@L> "False" => ast::Expr::new(location, ast::ExprKind::Constant { value: false.into(), kind: None }),
-    <location:@L> "None" => ast::Expr::new(location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }),
-    <location:@L> "..." => ast::Expr::new(location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }),
+    <location:@L> "True" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: true.into(), kind: None }),
+    <location:@L> "False" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: false.into(), kind: None }),
+    <location:@L> "None" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }),
+    <location:@L> "..." <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }),
 };
 
 ListLiteralValues: Vec<ast::Expr> = {
@@ -1141,12 +1202,13 @@ TestList: ast::Expr = {
 };
 
 GenericList<Element>: ast::Expr = {
-    <location:@L> <elts:OneOrMore<Element>> <trailing_comma:","?> => {
+    <location:@L> <elts:OneOrMore<Element>> <trailing_comma:","?> <end_location:@R> => {
         if elts.len() == 1 && trailing_comma.is_none() {
             elts.into_iter().next().unwrap()
         } else {
             ast::Expr {
                 location,
+                end_location: Some(end_location),
                 custom: (),
                 node: ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load }
             }
@@ -1156,8 +1218,9 @@ GenericList<Element>: ast::Expr = {
 
 // Test
 StarExpr: ast::Expr = {
-    <location:@L> "*" <e:Expression> => ast::Expr {
+    <location:@L> "*" <e:Expression> <end_location:@R> => ast::Expr {
         location,
+        end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Starred { value: Box::new(e), ctx: ast::ExprContext::Load },
     }
@@ -1167,7 +1230,7 @@ StarExpr: ast::Expr = {
 CompFor: Vec<ast::Comprehension> = <c:SingleForComprehension+> => c;
 
 SingleForComprehension: ast::Comprehension = {
-    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest> <ifs:ComprehensionIf*> => {
+    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest> <ifs:ComprehensionIf*> <end_location:@R> => {
         let is_async = is_async.is_some();
         ast::Comprehension {
             target: Box::new(target),
@@ -1188,11 +1251,12 @@ ArgumentList: ArgumentList = {
     }
 };
 
-FunctionArgument: (Option<(ast::Location, Option<String>)>, ast::Expr) = {
+FunctionArgument: (Option<(ast::Location, ast::Location, Option<String>)>, ast::Expr) = {
     <e:NamedExpressionTest> <c:CompFor?> => {
         let expr = match c {
             Some(c) => ast::Expr {
                 location: e.location,
+                end_location: e.end_location,
                 custom: (),
                 node: ast::ExprKind::GeneratorExp {
                     elt: Box::new(e),
@@ -1203,15 +1267,16 @@ FunctionArgument: (Option<(ast::Location, Option<String>)>, ast::Expr) = {
         };
         (None, expr)
     },
-    <location:@L> <i:Identifier> "=" <e:Test> => (Some((location, Some(i))), e),
-    <location:@L> "*" <e:Test> => {
+    <location:@L> <i:Identifier> "=" <e:Test> <end_location:@R> => (Some((location, end_location, Some(i))), e),
+    <location:@L> "*" <e:Test> <end_location:@R> => {
         let expr = ast::Expr::new(
             location,
+            end_location,
             ast::ExprKind::Starred { value: Box::new(e), ctx: ast::ExprContext::Load },
         );
         (None, expr)
     },
-    <location:@L> "**" <e:Test> => (Some((location, None)), e),
+    <location:@L> "**" <e:Test> <end_location:@R> => (Some((location, end_location, None)), e),
 };
 
 #[inline]
diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index 342e47de..23265831 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -7,17 +7,18 @@ use crate::{
 use std::{iter, mem, str};
 
 struct FStringParser {
-    str_location: Location,
+    str_start: Location,
+    str_end: Location,
 }
 
 impl FStringParser {
-    fn new(str_location: Location) -> Self {
-        Self { str_location }
+    fn new(str_start: Location, str_end: Location) -> Self {
+        Self { str_start, str_end }
     }
 
     #[inline]
     fn expr(&self, node: ExprKind) -> Expr {
-        Expr::new(self.str_location, node)
+        Expr::new(self.str_start, self.str_end, node)
     }
 
     fn parse_formatted_value<'a>(
@@ -305,11 +306,18 @@ fn parse_fstring_expr(source: &str) -> Result<Expr, ParseError> {
 
 /// Parse an fstring from a string, located at a certain position in the sourcecode.
 /// In case of errors, we will get the location and the error returned.
-pub fn parse_located_fstring(source: &str, location: Location) -> Result<Vec<Expr>, FStringError> {
-    FStringParser::new(location)
+pub fn parse_located_fstring(
+    source: &str,
+    start: Location,
+    end: Location,
+) -> Result<Vec<Expr>, FStringError> {
+    FStringParser::new(start, end)
         .parse(source.chars().peekable(), 0)
         .map(|(e, _)| e)
-        .map_err(|error| FStringError { error, location })
+        .map_err(|error| FStringError {
+            error,
+            location: start,
+        })
 }
 
 #[cfg(test)]
@@ -317,7 +325,7 @@ mod tests {
     use super::*;
 
     fn parse_fstring(source: &str) -> Result<Vec<Expr>, FStringErrorType> {
-        FStringParser::new(Location::default())
+        FStringParser::new(Location::default(), Location::default())
             .parse(source.chars().peekable(), 0)
             .map(|(e, _)| e)
     }
diff --git a/parser/src/function.rs b/parser/src/function.rs
index 2a384ddf..28cb44fe 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -45,7 +45,10 @@ pub fn parse_params(
     Ok((posonly, names, defaults))
 }
 
-type FunctionArgument = (Option<(ast::Location, Option<String>)>, ast::Expr);
+type FunctionArgument = (
+    Option<(ast::Location, ast::Location, Option<String>)>,
+    ast::Expr,
+);
 
 pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, LexicalError> {
     let mut args = vec![];
@@ -54,12 +57,12 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, Lexi
     let mut keyword_names = HashSet::with_capacity_and_hasher(func_args.len(), RandomState::new());
     for (name, value) in func_args {
         match name {
-            Some((location, name)) => {
+            Some((start, end, name)) => {
                 if let Some(keyword_name) = &name {
                     if keyword_names.contains(keyword_name) {
                         return Err(LexicalError {
                             error: LexicalErrorType::DuplicateKeywordArgumentError,
-                            location,
+                            location: start,
                         });
                     }
 
@@ -67,7 +70,8 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, Lexi
                 }
 
                 keywords.push(ast::Keyword::new(
-                    location,
+                    start,
+                    end,
                     ast::KeywordData {
                         arg: name,
                         value: Box::new(value),
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index cd74856c..49d2e53f 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -35,10 +35,12 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
 ///     expr,
 ///     ast::Expr {
 ///         location: ast::Location::new(1, 3),
+///         end_location: Some(ast::Location::new(1, 6)),
 ///         custom: (),
 ///         node: ast::ExprKind::BinOp {
 ///             left: Box::new(ast::Expr {
 ///                 location: ast::Location::new(1, 1),
+///                 end_location: Some(ast::Location::new(1, 2)),
 ///                 custom: (),
 ///                 node: ast::ExprKind::Constant {
 ///                     value: ast::Constant::Int(1.into()),
@@ -48,6 +50,7 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
 ///             op: ast::Operator::Add,
 ///             right: Box::new(ast::Expr {
 ///                 location: ast::Location::new(1, 5),
+///                 end_location: Some(ast::Location::new(1, 6)),
 ///                 custom: (),
 ///                 node: ast::ExprKind::Constant {
 ///                     value: ast::Constant::Int(2.into()),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__ann_assign_name.snap b/parser/src/snapshots/rustpython_parser__context__tests__ann_assign_name.snap
new file mode 100644
index 00000000..03b7e437
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__ann_assign_name.snap
@@ -0,0 +1,69 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 11,
+        },
+        custom: (),
+        node: AnnAssign {
+            target: Located {
+                start: Location {
+                    row: 1,
+                    column: 1,
+                },
+                end: Location {
+                    row: 1,
+                    column: 2,
+                },
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Store,
+                },
+            },
+            annotation: Located {
+                start: Location {
+                    row: 1,
+                    column: 4,
+                },
+                end: Location {
+                    row: 1,
+                    column: 7,
+                },
+                custom: (),
+                node: Name {
+                    id: "int",
+                    ctx: Load,
+                },
+            },
+            value: Some(
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 10,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 11,
+                    },
+                    custom: (),
+                    node: Constant {
+                        value: Int(
+                            1,
+                        ),
+                        kind: None,
+                    },
+                },
+            ),
+            simple: 1,
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap
new file mode 100644
index 00000000..9f3abe15
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap
@@ -0,0 +1,119 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 16,
+        },
+        custom: (),
+        node: Assign {
+            targets: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 2,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 4,
+                    },
+                    custom: (),
+                    node: Attribute {
+                        value: Located {
+                            start: Location {
+                                row: 1,
+                                column: 1,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 2,
+                            },
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Load,
+                            },
+                        },
+                        attr: "y",
+                        ctx: Store,
+                    },
+                },
+            ],
+            value: Located {
+                start: Location {
+                    row: 1,
+                    column: 8,
+                },
+                end: Location {
+                    row: 1,
+                    column: 15,
+                },
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 8,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 9,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 11,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 14,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 15,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    3,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+            type_comment: None,
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap
new file mode 100644
index 00000000..22d92643
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap
@@ -0,0 +1,117 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 25,
+        },
+        custom: (),
+        node: For {
+            target: Located {
+                start: Location {
+                    row: 1,
+                    column: 5,
+                },
+                end: Location {
+                    row: 1,
+                    column: 6,
+                },
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Store,
+                },
+            },
+            iter: Located {
+                start: Location {
+                    row: 1,
+                    column: 11,
+                },
+                end: Location {
+                    row: 1,
+                    column: 18,
+                },
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 11,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 14,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 15,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 17,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 18,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    3,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+            body: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 21,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 25,
+                    },
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            orelse: [],
+            type_comment: None,
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap
new file mode 100644
index 00000000..17abaae4
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap
@@ -0,0 +1,135 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 19,
+        },
+        custom: (),
+        node: Assign {
+            targets: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 1,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 7,
+                    },
+                    custom: (),
+                    node: List {
+                        elts: [
+                            Located {
+                                start: Location {
+                                    row: 1,
+                                    column: 2,
+                                },
+                                end: Location {
+                                    row: 1,
+                                    column: 3,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "x",
+                                    ctx: Store,
+                                },
+                            },
+                            Located {
+                                start: Location {
+                                    row: 1,
+                                    column: 5,
+                                },
+                                end: Location {
+                                    row: 1,
+                                    column: 6,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "y",
+                                    ctx: Store,
+                                },
+                            },
+                        ],
+                        ctx: Store,
+                    },
+                },
+            ],
+            value: Located {
+                start: Location {
+                    row: 1,
+                    column: 11,
+                },
+                end: Location {
+                    row: 1,
+                    column: 18,
+                },
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 11,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 14,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 15,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 17,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 18,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    3,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+            type_comment: None,
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap
new file mode 100644
index 00000000..929f6695
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap
@@ -0,0 +1,153 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 27,
+        },
+        custom: (),
+        node: Assign {
+            targets: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 1,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 2,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "x",
+                        ctx: Store,
+                    },
+                },
+            ],
+            value: Located {
+                start: Location {
+                    row: 1,
+                    column: 5,
+                },
+                end: Location {
+                    row: 1,
+                    column: 27,
+                },
+                custom: (),
+                node: ListComp {
+                    elt: Located {
+                        start: Location {
+                            row: 1,
+                            column: 6,
+                        },
+                        end: Location {
+                            row: 1,
+                            column: 7,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "y",
+                            ctx: Load,
+                        },
+                    },
+                    generators: [
+                        Comprehension {
+                            target: Located {
+                                start: Location {
+                                    row: 1,
+                                    column: 12,
+                                },
+                                end: Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "y",
+                                    ctx: Store,
+                                },
+                            },
+                            iter: Located {
+                                start: Location {
+                                    row: 1,
+                                    column: 18,
+                                },
+                                end: Location {
+                                    row: 1,
+                                    column: 25,
+                                },
+                                custom: (),
+                                node: Tuple {
+                                    elts: [
+                                        Located {
+                                            start: Location {
+                                                row: 1,
+                                                column: 18,
+                                            },
+                                            end: Location {
+                                                row: 1,
+                                                column: 19,
+                                            },
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    1,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                        Located {
+                                            start: Location {
+                                                row: 1,
+                                                column: 21,
+                                            },
+                                            end: Location {
+                                                row: 1,
+                                                column: 22,
+                                            },
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    2,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                        Located {
+                                            start: Location {
+                                                row: 1,
+                                                column: 24,
+                                            },
+                                            end: Location {
+                                                row: 1,
+                                                column: 25,
+                                            },
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    3,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    ],
+                                    ctx: Load,
+                                },
+                            },
+                            ifs: [],
+                            is_async: 0,
+                        },
+                    ],
+                },
+            },
+            type_comment: None,
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap
new file mode 100644
index 00000000..614b04ad
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap
@@ -0,0 +1,104 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 14,
+        },
+        custom: (),
+        node: Assign {
+            targets: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 1,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 2,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "x",
+                        ctx: Store,
+                    },
+                },
+            ],
+            value: Located {
+                start: Location {
+                    row: 1,
+                    column: 6,
+                },
+                end: Location {
+                    row: 1,
+                    column: 13,
+                },
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 6,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 7,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 9,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 10,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 13,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    3,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+            type_comment: None,
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap
new file mode 100644
index 00000000..7c9559a8
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap
@@ -0,0 +1,79 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 15,
+        },
+        custom: (),
+        node: If {
+            test: Located {
+                start: Location {
+                    row: 1,
+                    column: 4,
+                },
+                end: Location {
+                    row: 1,
+                    column: 9,
+                },
+                custom: (),
+                node: NamedExpr {
+                    target: Located {
+                        start: Location {
+                            row: 1,
+                            column: 4,
+                        },
+                        end: Location {
+                            row: 1,
+                            column: 9,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "x",
+                            ctx: Store,
+                        },
+                    },
+                    value: Located {
+                        start: Location {
+                            row: 1,
+                            column: 8,
+                        },
+                        end: Location {
+                            row: 1,
+                            column: 9,
+                        },
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                1,
+                            ),
+                            kind: None,
+                        },
+                    },
+                },
+            },
+            body: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 11,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 15,
+                    },
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            orelse: [],
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap
new file mode 100644
index 00000000..abf7db89
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap
@@ -0,0 +1,153 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 27,
+        },
+        custom: (),
+        node: Assign {
+            targets: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 1,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 2,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "x",
+                        ctx: Store,
+                    },
+                },
+            ],
+            value: Located {
+                start: Location {
+                    row: 1,
+                    column: 5,
+                },
+                end: Location {
+                    row: 1,
+                    column: 27,
+                },
+                custom: (),
+                node: SetComp {
+                    elt: Located {
+                        start: Location {
+                            row: 1,
+                            column: 6,
+                        },
+                        end: Location {
+                            row: 1,
+                            column: 7,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "y",
+                            ctx: Load,
+                        },
+                    },
+                    generators: [
+                        Comprehension {
+                            target: Located {
+                                start: Location {
+                                    row: 1,
+                                    column: 12,
+                                },
+                                end: Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "y",
+                                    ctx: Store,
+                                },
+                            },
+                            iter: Located {
+                                start: Location {
+                                    row: 1,
+                                    column: 18,
+                                },
+                                end: Location {
+                                    row: 1,
+                                    column: 25,
+                                },
+                                custom: (),
+                                node: Tuple {
+                                    elts: [
+                                        Located {
+                                            start: Location {
+                                                row: 1,
+                                                column: 18,
+                                            },
+                                            end: Location {
+                                                row: 1,
+                                                column: 19,
+                                            },
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    1,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                        Located {
+                                            start: Location {
+                                                row: 1,
+                                                column: 21,
+                                            },
+                                            end: Location {
+                                                row: 1,
+                                                column: 22,
+                                            },
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    2,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                        Located {
+                                            start: Location {
+                                                row: 1,
+                                                column: 24,
+                                            },
+                                            end: Location {
+                                                row: 1,
+                                                column: 25,
+                                            },
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    3,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    ],
+                                    ctx: Load,
+                                },
+                            },
+                            ifs: [],
+                            is_async: 0,
+                        },
+                    ],
+                },
+            },
+            type_comment: None,
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap
new file mode 100644
index 00000000..24c062e2
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap
@@ -0,0 +1,149 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 20,
+        },
+        custom: (),
+        node: Assign {
+            targets: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 2,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 7,
+                    },
+                    custom: (),
+                    node: Tuple {
+                        elts: [
+                            Located {
+                                start: Location {
+                                    row: 1,
+                                    column: 2,
+                                },
+                                end: Location {
+                                    row: 1,
+                                    column: 3,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "x",
+                                    ctx: Store,
+                                },
+                            },
+                            Located {
+                                start: Location {
+                                    row: 1,
+                                    column: 5,
+                                },
+                                end: Location {
+                                    row: 1,
+                                    column: 7,
+                                },
+                                custom: (),
+                                node: Starred {
+                                    value: Located {
+                                        start: Location {
+                                            row: 1,
+                                            column: 6,
+                                        },
+                                        end: Location {
+                                            row: 1,
+                                            column: 7,
+                                        },
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                    ctx: Store,
+                                },
+                            },
+                        ],
+                        ctx: Store,
+                    },
+                },
+            ],
+            value: Located {
+                start: Location {
+                    row: 1,
+                    column: 12,
+                },
+                end: Location {
+                    row: 1,
+                    column: 19,
+                },
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 13,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 15,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 16,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 18,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 19,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    3,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+            type_comment: None,
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap
new file mode 100644
index 00000000..7028f2ef
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap
@@ -0,0 +1,133 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 17,
+        },
+        custom: (),
+        node: Assign {
+            targets: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 2,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 5,
+                    },
+                    custom: (),
+                    node: Subscript {
+                        value: Located {
+                            start: Location {
+                                row: 1,
+                                column: 1,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 2,
+                            },
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Load,
+                            },
+                        },
+                        slice: Located {
+                            start: Location {
+                                row: 1,
+                                column: 3,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 4,
+                            },
+                            custom: (),
+                            node: Name {
+                                id: "y",
+                                ctx: Load,
+                            },
+                        },
+                        ctx: Store,
+                    },
+                },
+            ],
+            value: Located {
+                start: Location {
+                    row: 1,
+                    column: 9,
+                },
+                end: Location {
+                    row: 1,
+                    column: 16,
+                },
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 9,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 10,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 13,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 15,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 16,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    3,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+            type_comment: None,
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap
new file mode 100644
index 00000000..0d9a1958
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap
@@ -0,0 +1,135 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 19,
+        },
+        custom: (),
+        node: Assign {
+            targets: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 2,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 6,
+                    },
+                    custom: (),
+                    node: Tuple {
+                        elts: [
+                            Located {
+                                start: Location {
+                                    row: 1,
+                                    column: 2,
+                                },
+                                end: Location {
+                                    row: 1,
+                                    column: 3,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "x",
+                                    ctx: Store,
+                                },
+                            },
+                            Located {
+                                start: Location {
+                                    row: 1,
+                                    column: 5,
+                                },
+                                end: Location {
+                                    row: 1,
+                                    column: 6,
+                                },
+                                custom: (),
+                                node: Name {
+                                    id: "y",
+                                    ctx: Store,
+                                },
+                            },
+                        ],
+                        ctx: Store,
+                    },
+                },
+            ],
+            value: Located {
+                start: Location {
+                    row: 1,
+                    column: 11,
+                },
+                end: Location {
+                    row: 1,
+                    column: 18,
+                },
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 11,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 14,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 15,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 17,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 18,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    3,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+            type_comment: None,
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_with.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_with.snap
new file mode 100644
index 00000000..042d56b5
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_with.snap
@@ -0,0 +1,72 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 18,
+        },
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        start: Location {
+                            row: 1,
+                            column: 6,
+                        },
+                        end: Location {
+                            row: 1,
+                            column: 7,
+                        },
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                1,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 11,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 14,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 18,
+                    },
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap
new file mode 100644
index 00000000..1e55a560
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap
@@ -0,0 +1,117 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 17,
+        },
+        custom: (),
+        node: AugAssign {
+            target: Located {
+                start: Location {
+                    row: 1,
+                    column: 2,
+                },
+                end: Location {
+                    row: 1,
+                    column: 4,
+                },
+                custom: (),
+                node: Attribute {
+                    value: Located {
+                        start: Location {
+                            row: 1,
+                            column: 1,
+                        },
+                        end: Location {
+                            row: 1,
+                            column: 2,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "x",
+                            ctx: Load,
+                        },
+                    },
+                    attr: "y",
+                    ctx: Store,
+                },
+            },
+            op: Add,
+            value: Located {
+                start: Location {
+                    row: 1,
+                    column: 9,
+                },
+                end: Location {
+                    row: 1,
+                    column: 16,
+                },
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 9,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 10,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 13,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 15,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 16,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    3,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_name.snap b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_name.snap
new file mode 100644
index 00000000..576c613f
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_name.snap
@@ -0,0 +1,52 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 7,
+        },
+        custom: (),
+        node: AugAssign {
+            target: Located {
+                start: Location {
+                    row: 1,
+                    column: 1,
+                },
+                end: Location {
+                    row: 1,
+                    column: 2,
+                },
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Store,
+                },
+            },
+            op: Add,
+            value: Located {
+                start: Location {
+                    row: 1,
+                    column: 6,
+                },
+                end: Location {
+                    row: 1,
+                    column: 7,
+                },
+                custom: (),
+                node: Constant {
+                    value: Int(
+                        1,
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap
new file mode 100644
index 00000000..881f665b
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap
@@ -0,0 +1,131 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 18,
+        },
+        custom: (),
+        node: AugAssign {
+            target: Located {
+                start: Location {
+                    row: 1,
+                    column: 2,
+                },
+                end: Location {
+                    row: 1,
+                    column: 5,
+                },
+                custom: (),
+                node: Subscript {
+                    value: Located {
+                        start: Location {
+                            row: 1,
+                            column: 1,
+                        },
+                        end: Location {
+                            row: 1,
+                            column: 2,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "x",
+                            ctx: Load,
+                        },
+                    },
+                    slice: Located {
+                        start: Location {
+                            row: 1,
+                            column: 3,
+                        },
+                        end: Location {
+                            row: 1,
+                            column: 4,
+                        },
+                        custom: (),
+                        node: Name {
+                            id: "y",
+                            ctx: Load,
+                        },
+                    },
+                    ctx: Store,
+                },
+            },
+            op: Add,
+            value: Located {
+                start: Location {
+                    row: 1,
+                    column: 10,
+                },
+                end: Location {
+                    row: 1,
+                    column: 17,
+                },
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 10,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 11,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 13,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 14,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            start: Location {
+                                row: 1,
+                                column: 16,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 17,
+                            },
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    3,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__del_attribute.snap b/parser/src/snapshots/rustpython_parser__context__tests__del_attribute.snap
new file mode 100644
index 00000000..39e650e2
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__del_attribute.snap
@@ -0,0 +1,51 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 8,
+        },
+        custom: (),
+        node: Delete {
+            targets: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 6,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 8,
+                    },
+                    custom: (),
+                    node: Attribute {
+                        value: Located {
+                            start: Location {
+                                row: 1,
+                                column: 5,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 6,
+                            },
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Load,
+                            },
+                        },
+                        attr: "y",
+                        ctx: Del,
+                    },
+                },
+            ],
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__del_name.snap b/parser/src/snapshots/rustpython_parser__context__tests__del_name.snap
new file mode 100644
index 00000000..396a5700
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__del_name.snap
@@ -0,0 +1,36 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 6,
+        },
+        custom: (),
+        node: Delete {
+            targets: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 5,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 6,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "x",
+                        ctx: Del,
+                    },
+                },
+            ],
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__del_subscript.snap b/parser/src/snapshots/rustpython_parser__context__tests__del_subscript.snap
new file mode 100644
index 00000000..56da6c49
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__del_subscript.snap
@@ -0,0 +1,65 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        start: Location {
+            row: 1,
+            column: 1,
+        },
+        end: Location {
+            row: 1,
+            column: 9,
+        },
+        custom: (),
+        node: Delete {
+            targets: [
+                Located {
+                    start: Location {
+                        row: 1,
+                        column: 6,
+                    },
+                    end: Location {
+                        row: 1,
+                        column: 9,
+                    },
+                    custom: (),
+                    node: Subscript {
+                        value: Located {
+                            start: Location {
+                                row: 1,
+                                column: 5,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 6,
+                            },
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Load,
+                            },
+                        },
+                        slice: Located {
+                            start: Location {
+                                row: 1,
+                                column: 7,
+                            },
+                            end: Location {
+                                row: 1,
+                                column: 8,
+                            },
+                            custom: (),
+                            node: Name {
+                                id: "y",
+                                ctx: Load,
+                            },
+                        },
+                        ctx: Del,
+                    },
+                },
+            ],
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
index 14e7003f..d54af22b 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -21,6 +27,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -34,6 +46,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: FormattedValue {
             value: Located {
@@ -41,6 +59,12 @@ expression: parse_ast
                     row: 1,
                     column: 2,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 6,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "user",
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
index 0df5ab2a..df3a617e 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -21,6 +27,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -34,6 +46,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -47,6 +65,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: FormattedValue {
             value: Located {
@@ -54,6 +78,12 @@ expression: parse_ast
                     row: 1,
                     column: 2,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 6,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "user",
@@ -69,6 +99,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -82,6 +118,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -95,6 +137,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -108,6 +156,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: FormattedValue {
             value: Located {
@@ -115,6 +169,12 @@ expression: parse_ast
                     row: 1,
                     column: 2,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 8,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "second",
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
index f5a518d7..a224dd43 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -21,6 +27,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -34,6 +46,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: FormattedValue {
             value: Located {
@@ -41,6 +59,12 @@ expression: parse_ast
                     row: 1,
                     column: 2,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 6,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "user",
@@ -54,6 +78,12 @@ expression: parse_ast
                         row: 0,
                         column: 0,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 0,
+                            column: 0,
+                        },
+                    ),
                     custom: (),
                     node: JoinedStr {
                         values: [
@@ -62,6 +92,12 @@ expression: parse_ast
                                     row: 0,
                                     column: 0,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 0,
+                                        column: 0,
+                                    },
+                                ),
                                 custom: (),
                                 node: Constant {
                                     value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
index 7210ed34..d81ee52d 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: FormattedValue {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 2,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 3,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "a",
@@ -30,6 +42,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: FormattedValue {
             value: Located {
@@ -37,6 +55,12 @@ expression: parse_ast
                     row: 1,
                     column: 3,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 4,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "b",
@@ -52,6 +76,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
index f9d6c4d2..3b50a46e 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: FormattedValue {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 5,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 10,
+                    },
+                ),
                 custom: (),
                 node: Compare {
                     left: Located {
@@ -22,6 +34,12 @@ expression: parse_ast
                             row: 1,
                             column: 2,
                         },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 4,
+                            },
+                        ),
                         custom: (),
                         node: Constant {
                             value: Int(
@@ -39,6 +57,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 8,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
index bb3f26af..a8d98238 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: FormattedValue {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 2,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 5,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "foo",
@@ -28,6 +40,12 @@ expression: parse_ast
                         row: 0,
                         column: 0,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 0,
+                            column: 0,
+                        },
+                    ),
                     custom: (),
                     node: JoinedStr {
                         values: [
@@ -36,6 +54,12 @@ expression: parse_ast
                                     row: 0,
                                     column: 0,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 0,
+                                        column: 0,
+                                    },
+                                ),
                                 custom: (),
                                 node: FormattedValue {
                                     value: Located {
@@ -43,6 +67,12 @@ expression: parse_ast
                                             row: 1,
                                             column: 2,
                                         },
+                                        end_location: Some(
+                                            Location {
+                                                row: 1,
+                                                column: 6,
+                                            },
+                                        ),
                                         custom: (),
                                         node: Name {
                                             id: "spec",
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
index f9223a59..4e0f68f8 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: FormattedValue {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 4,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 8,
+                    },
+                ),
                 custom: (),
                 node: Compare {
                     left: Located {
@@ -22,6 +34,12 @@ expression: parse_ast
                             row: 1,
                             column: 2,
                         },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 3,
+                            },
+                        ),
                         custom: (),
                         node: Constant {
                             value: Int(
@@ -39,6 +57,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 7,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 8,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
index e15ac253..c4517c35 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: FormattedValue {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 2,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 5,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "foo",
@@ -28,6 +40,12 @@ expression: parse_ast
                         row: 0,
                         column: 0,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 0,
+                            column: 0,
+                        },
+                    ),
                     custom: (),
                     node: JoinedStr {
                         values: [
@@ -36,6 +54,12 @@ expression: parse_ast
                                     row: 0,
                                     column: 0,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 0,
+                                        column: 0,
+                                    },
+                                ),
                                 custom: (),
                                 node: Constant {
                                     value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
index ae998f0a..58efec1c 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -21,6 +27,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -34,6 +46,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: FormattedValue {
             value: Located {
@@ -41,6 +59,12 @@ expression: parse_ast
                     row: 1,
                     column: 2,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 3,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "x",
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
index 317bded0..d9d49f2a 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -21,6 +27,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: Constant {
             value: Str(
@@ -34,6 +46,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: FormattedValue {
             value: Located {
@@ -41,6 +59,12 @@ expression: parse_ast
                     row: 1,
                     column: 2,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 3,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "x",
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
index cb7d7208..55aa0d18 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 0,
             column: 0,
         },
+        end_location: Some(
+            Location {
+                row: 0,
+                column: 0,
+            },
+        ),
         custom: (),
         node: FormattedValue {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 2,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 7,
+                    },
+                ),
                 custom: (),
                 node: Yield {
                     value: None,
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
index 327da516..5797f7f2 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
@@ -8,6 +8,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 5,
+                column: 7,
+            },
+        ),
         custom: (),
         node: ClassDef {
             name: "Foo",
@@ -17,6 +23,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                         row: 1,
                         column: 11,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 12,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "A",
@@ -28,6 +40,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                         row: 1,
                         column: 14,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 15,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "B",
@@ -42,6 +60,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                         row: 2,
                         column: 2,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 4,
+                            column: 2,
+                        },
+                    ),
                     custom: (),
                     node: FunctionDef {
                         name: "__init__",
@@ -53,6 +77,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                         row: 2,
                                         column: 15,
                                     },
+                                    end_location: Some(
+                                        Location {
+                                            row: 2,
+                                            column: 19,
+                                        },
+                                    ),
                                     custom: (),
                                     node: ArgData {
                                         arg: "self",
@@ -73,6 +103,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                     row: 3,
                                     column: 3,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 3,
+                                        column: 7,
+                                    },
+                                ),
                                 custom: (),
                                 node: Pass,
                             },
@@ -87,6 +123,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                         row: 4,
                         column: 2,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 5,
+                            column: 7,
+                        },
+                    ),
                     custom: (),
                     node: FunctionDef {
                         name: "method_with_default",
@@ -98,6 +140,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                         row: 4,
                                         column: 26,
                                     },
+                                    end_location: Some(
+                                        Location {
+                                            row: 4,
+                                            column: 30,
+                                        },
+                                    ),
                                     custom: (),
                                     node: ArgData {
                                         arg: "self",
@@ -110,6 +158,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                         row: 4,
                                         column: 32,
                                     },
+                                    end_location: Some(
+                                        Location {
+                                            row: 4,
+                                            column: 35,
+                                        },
+                                    ),
                                     custom: (),
                                     node: ArgData {
                                         arg: "arg",
@@ -128,6 +182,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                         row: 4,
                                         column: 36,
                                     },
+                                    end_location: Some(
+                                        Location {
+                                            row: 4,
+                                            column: 45,
+                                        },
+                                    ),
                                     custom: (),
                                     node: Constant {
                                         value: Str(
@@ -144,6 +204,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                     row: 5,
                                     column: 3,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 5,
+                                        column: 7,
+                                    },
+                                ),
                                 custom: (),
                                 node: Pass,
                             },
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap
index f7f9b52d..a75ccdce 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
 ---
 Located {
@@ -7,6 +7,12 @@ Located {
         row: 1,
         column: 1,
     },
+    end_location: Some(
+        Location {
+            row: 1,
+            column: 20,
+        },
+    ),
     custom: (),
     node: DictComp {
         key: Located {
@@ -14,6 +20,12 @@ Located {
                 row: 1,
                 column: 2,
             },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 4,
+                },
+            ),
             custom: (),
             node: Name {
                 id: "x1",
@@ -25,6 +37,12 @@ Located {
                 row: 1,
                 column: 6,
             },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 8,
+                },
+            ),
             custom: (),
             node: Name {
                 id: "x2",
@@ -38,6 +56,12 @@ Located {
                         row: 1,
                         column: 13,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 14,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "y",
@@ -49,6 +73,12 @@ Located {
                         row: 1,
                         column: 18,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 19,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "z",
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
index 52d4b6ce..3018e3ff 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
 ---
 Located {
@@ -7,6 +7,12 @@ Located {
         row: 1,
         column: 1,
     },
+    end_location: Some(
+        Location {
+            row: 1,
+            column: 49,
+        },
+    ),
     custom: (),
     node: ListComp {
         elt: Located {
@@ -14,6 +20,12 @@ Located {
                 row: 1,
                 column: 2,
             },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 3,
+                },
+            ),
             custom: (),
             node: Name {
                 id: "x",
@@ -27,6 +39,12 @@ Located {
                         row: 1,
                         column: 8,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 13,
+                        },
+                    ),
                     custom: (),
                     node: Tuple {
                         elts: [
@@ -35,6 +53,12 @@ Located {
                                     row: 1,
                                     column: 8,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 9,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "y",
@@ -46,6 +70,12 @@ Located {
                                     row: 1,
                                     column: 11,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 13,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "y2",
@@ -61,6 +91,12 @@ Located {
                         row: 1,
                         column: 17,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 18,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "z",
@@ -76,6 +112,12 @@ Located {
                         row: 1,
                         column: 23,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 24,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "a",
@@ -87,6 +129,12 @@ Located {
                         row: 1,
                         column: 28,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 29,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "b",
@@ -99,6 +147,12 @@ Located {
                             row: 1,
                             column: 35,
                         },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 38,
+                            },
+                        ),
                         custom: (),
                         node: Compare {
                             left: Located {
@@ -106,6 +160,12 @@ Located {
                                     row: 1,
                                     column: 33,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 34,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "a",
@@ -121,6 +181,12 @@ Located {
                                         row: 1,
                                         column: 37,
                                     },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 38,
+                                        },
+                                    ),
                                     custom: (),
                                     node: Constant {
                                         value: Int(
@@ -137,6 +203,12 @@ Located {
                             row: 1,
                             column: 44,
                         },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 48,
+                            },
+                        ),
                         custom: (),
                         node: Compare {
                             left: Located {
@@ -144,6 +216,12 @@ Located {
                                     row: 1,
                                     column: 42,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 43,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "a",
@@ -159,6 +237,12 @@ Located {
                                         row: 1,
                                         column: 46,
                                     },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 48,
+                                        },
+                                    ),
                                     custom: (),
                                     node: Constant {
                                         value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap
index 97b49bdd..13c2fc69 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 15,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 15,
+                    },
+                ),
                 custom: (),
                 node: JoinedStr {
                     values: [
@@ -23,6 +35,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 1,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 15,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap
index 0e037887..0b85a1ab 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
 ---
 Located {
@@ -7,6 +7,12 @@ Located {
         row: 1,
         column: 1,
     },
+    end_location: Some(
+        Location {
+            row: 1,
+            column: 15,
+        },
+    ),
     custom: (),
     node: GeneratorExp {
         elt: Located {
@@ -14,6 +20,12 @@ Located {
                 row: 1,
                 column: 2,
             },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 3,
+                },
+            ),
             custom: (),
             node: Name {
                 id: "x",
@@ -27,6 +39,12 @@ Located {
                         row: 1,
                         column: 8,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 9,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "y",
@@ -38,6 +56,12 @@ Located {
                         row: 1,
                         column: 13,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 14,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "z",
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
index 4b34c0e4..c628cf98 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
@@ -1,7 +1,6 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
-
 ---
 [
     Located {
@@ -9,6 +8,12 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 3,
+                column: 9,
+            },
+        ),
         custom: (),
         node: If {
             test: Located {
@@ -16,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 4,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 5,
+                    },
+                ),
                 custom: (),
                 node: Constant {
                     value: Int(
@@ -30,6 +41,12 @@ expression: parse_ast
                         row: 1,
                         column: 7,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 9,
+                        },
+                    ),
                     custom: (),
                     node: Expr {
                         value: Located {
@@ -37,6 +54,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 7,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 9,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -54,6 +77,12 @@ expression: parse_ast
                         row: 2,
                         column: 1,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 3,
+                            column: 1,
+                        },
+                    ),
                     custom: (),
                     node: If {
                         test: Located {
@@ -61,6 +90,12 @@ expression: parse_ast
                                 row: 2,
                                 column: 6,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 7,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -75,6 +110,12 @@ expression: parse_ast
                                     row: 2,
                                     column: 9,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 2,
+                                        column: 11,
+                                    },
+                                ),
                                 custom: (),
                                 node: Expr {
                                     value: Located {
@@ -82,6 +123,12 @@ expression: parse_ast
                                             row: 2,
                                             column: 9,
                                         },
+                                        end_location: Some(
+                                            Location {
+                                                row: 2,
+                                                column: 11,
+                                            },
+                                        ),
                                         custom: (),
                                         node: Constant {
                                             value: Int(
@@ -99,6 +146,12 @@ expression: parse_ast
                                     row: 3,
                                     column: 7,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 3,
+                                        column: 9,
+                                    },
+                                ),
                                 custom: (),
                                 node: Expr {
                                     value: Located {
@@ -106,6 +159,12 @@ expression: parse_ast
                                             row: 3,
                                             column: 7,
                                         },
+                                        end_location: Some(
+                                            Location {
+                                                row: 3,
+                                                column: 9,
+                                            },
+                                        ),
                                         custom: (),
                                         node: Constant {
                                             value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
index 782df39a..4505d92c 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
 ---
 Located {
@@ -7,6 +7,12 @@ Located {
         row: 1,
         column: 1,
     },
+    end_location: Some(
+        Location {
+            row: 1,
+            column: 27,
+        },
+    ),
     custom: (),
     node: GeneratorExp {
         elt: Located {
@@ -14,6 +20,12 @@ Located {
                 row: 1,
                 column: 4,
             },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 15,
+                },
+            ),
             custom: (),
             node: IfExp {
                 test: Located {
@@ -21,6 +33,12 @@ Located {
                         row: 1,
                         column: 7,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 8,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "y",
@@ -32,6 +50,12 @@ Located {
                         row: 1,
                         column: 2,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 3,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "x",
@@ -43,6 +67,12 @@ Located {
                         row: 1,
                         column: 14,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 15,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "y",
@@ -58,6 +88,12 @@ Located {
                         row: 1,
                         column: 20,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 21,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "y",
@@ -69,6 +105,12 @@ Located {
                         row: 1,
                         column: 25,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 26,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "z",
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap
index 1b8a5451..61d45da8 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap
@@ -8,13 +8,25 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 33,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 8,
+                    column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 33,
+                    },
+                ),
                 custom: (),
                 node: Call {
                     func: Located {
@@ -22,6 +34,12 @@ expression: parse_ast
                             row: 1,
                             column: 1,
                         },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 8,
+                            },
+                        ),
                         custom: (),
                         node: Name {
                             id: "my_func",
@@ -34,6 +52,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 9,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 21,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Str(
@@ -49,6 +73,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 23,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 32,
+                                },
+                            ),
                             custom: (),
                             node: KeywordData {
                                 arg: Some(
@@ -59,6 +89,12 @@ expression: parse_ast
                                         row: 1,
                                         column: 31,
                                     },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 32,
+                                        },
+                                    ),
                                     custom: (),
                                     node: Constant {
                                         value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap
index 2f70292d..7e0395a4 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
 ---
 [
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 19,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 19,
+                    },
+                ),
                 custom: (),
                 node: Lambda {
                     args: Arguments {
@@ -25,6 +37,12 @@ expression: parse_ast
                                     row: 1,
                                     column: 8,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 9,
+                                    },
+                                ),
                                 custom: (),
                                 node: ArgData {
                                     arg: "x",
@@ -37,6 +55,12 @@ expression: parse_ast
                                     row: 1,
                                     column: 11,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 12,
+                                    },
+                                ),
                                 custom: (),
                                 node: ArgData {
                                     arg: "y",
@@ -56,6 +80,12 @@ expression: parse_ast
                             row: 1,
                             column: 16,
                         },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 19,
+                            },
+                        ),
                         custom: (),
                         node: BinOp {
                             left: Located {
@@ -63,6 +93,12 @@ expression: parse_ast
                                     row: 1,
                                     column: 14,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 15,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "x",
@@ -75,6 +111,12 @@ expression: parse_ast
                                     row: 1,
                                     column: 18,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 19,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "y",
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap
index 8a6d8eb8..d6916653 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
 ---
 Located {
@@ -7,6 +7,12 @@ Located {
         row: 1,
         column: 1,
     },
+    end_location: Some(
+        Location {
+            row: 1,
+            column: 15,
+        },
+    ),
     custom: (),
     node: ListComp {
         elt: Located {
@@ -14,6 +20,12 @@ Located {
                 row: 1,
                 column: 2,
             },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 3,
+                },
+            ),
             custom: (),
             node: Name {
                 id: "x",
@@ -27,6 +39,12 @@ Located {
                         row: 1,
                         column: 8,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 9,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "y",
@@ -38,6 +56,12 @@ Located {
                         row: 1,
                         column: 13,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 14,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "z",
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
index 960151b0..660f0fac 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
 ---
 Located {
@@ -7,6 +7,12 @@ Located {
         row: 1,
         column: 1,
     },
+    end_location: Some(
+        Location {
+            row: 1,
+            column: 24,
+        },
+    ),
     custom: (),
     node: GeneratorExp {
         elt: Located {
@@ -14,6 +20,12 @@ Located {
                 row: 1,
                 column: 2,
             },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 12,
+                },
+            ),
             custom: (),
             node: NamedExpr {
                 target: Located {
@@ -21,6 +33,12 @@ Located {
                         row: 1,
                         column: 2,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 12,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "x",
@@ -32,6 +50,12 @@ Located {
                         row: 1,
                         column: 9,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 12,
+                        },
+                    ),
                     custom: (),
                     node: BinOp {
                         left: Located {
@@ -39,6 +63,12 @@ Located {
                                 row: 1,
                                 column: 7,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 8,
+                                },
+                            ),
                             custom: (),
                             node: Name {
                                 id: "y",
@@ -51,6 +81,12 @@ Located {
                                 row: 1,
                                 column: 11,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 12,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -70,6 +106,12 @@ Located {
                         row: 1,
                         column: 17,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 18,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "y",
@@ -81,6 +123,12 @@ Located {
                         row: 1,
                         column: 22,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 23,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "z",
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap
index 3f828ded..fcb2eecc 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap
@@ -8,13 +8,25 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 24,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 6,
+                    column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 24,
+                    },
+                ),
                 custom: (),
                 node: Call {
                     func: Located {
@@ -22,6 +34,12 @@ expression: parse_ast
                             row: 1,
                             column: 1,
                         },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 6,
+                            },
+                        ),
                         custom: (),
                         node: Name {
                             id: "print",
@@ -34,6 +52,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 7,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 20,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Str(
@@ -47,6 +71,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 22,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 23,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap
index aae345a0..49efd477 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap
@@ -8,13 +8,25 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 21,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 6,
+                    column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 21,
+                    },
+                ),
                 custom: (),
                 node: Call {
                     func: Located {
@@ -22,6 +34,12 @@ expression: parse_ast
                             row: 1,
                             column: 1,
                         },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 6,
+                            },
+                        ),
                         custom: (),
                         node: Name {
                             id: "print",
@@ -34,6 +52,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 7,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 20,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap
index efbf32fd..318a0f9c 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 14,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 14,
+                    },
+                ),
                 custom: (),
                 node: Constant {
                     value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap
index 26243f56..d50b57d7 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap
@@ -1,7 +1,6 @@
 ---
-source: parser/src/parser.rs
-expression: parse_program(&source).unwrap()
-
+source: compiler/parser/src/parser.rs
+expression: "parse_program(source, \"<test>\").unwrap()"
 ---
 [
     Located {
@@ -9,6 +8,12 @@ expression: parse_program(&source).unwrap()
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 12,
+            },
+        ),
         custom: (),
         node: Assign {
             targets: [
@@ -17,6 +22,12 @@ expression: parse_program(&source).unwrap()
                         row: 1,
                         column: 1,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 5,
+                        },
+                    ),
                     custom: (),
                     node: Tuple {
                         elts: [
@@ -25,6 +36,12 @@ expression: parse_program(&source).unwrap()
                                     row: 1,
                                     column: 1,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 2,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "a",
@@ -36,6 +53,12 @@ expression: parse_program(&source).unwrap()
                                     row: 1,
                                     column: 4,
                                 },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 5,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "b",
@@ -52,6 +75,12 @@ expression: parse_program(&source).unwrap()
                     row: 1,
                     column: 8,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 12,
+                    },
+                ),
                 custom: (),
                 node: Tuple {
                     elts: [
@@ -60,6 +89,12 @@ expression: parse_program(&source).unwrap()
                                 row: 1,
                                 column: 8,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 9,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -73,6 +108,12 @@ expression: parse_program(&source).unwrap()
                                 row: 1,
                                 column: 11,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 12,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
index a486e11a..3cce5b1d 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 18,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 9,
+                    },
+                ),
                 custom: (),
                 node: JoinedStr {
                     values: [
@@ -23,6 +35,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 1,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 9,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
index a486e11a..3cce5b1d 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 18,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 9,
+                    },
+                ),
                 custom: (),
                 node: JoinedStr {
                     values: [
@@ -23,6 +35,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 1,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 9,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
index ccddd2ad..2ab56927 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 23,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 9,
+                    },
+                ),
                 custom: (),
                 node: JoinedStr {
                     values: [
@@ -23,6 +35,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 1,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 9,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Str(
@@ -36,6 +54,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 10,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 23,
+                                },
+                            ),
                             custom: (),
                             node: FormattedValue {
                                 value: Located {
@@ -43,6 +67,12 @@ expression: parse_ast
                                         row: 1,
                                         column: 2,
                                     },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 5,
+                                        },
+                                    ),
                                     custom: (),
                                     node: Constant {
                                         value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
index 380a1fc8..fd88e7b3 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 17,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 9,
+                    },
+                ),
                 custom: (),
                 node: Constant {
                     value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap
index 66db2319..a08dde85 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 21,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 21,
+                    },
+                ),
                 custom: (),
                 node: Constant {
                     value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
index dd6d3388..ba2ae385 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 19,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 10,
+                    },
+                ),
                 custom: (),
                 node: JoinedStr {
                     values: [
@@ -23,6 +35,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 1,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
index b5d6f940..7d21fea8 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 23,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 10,
+                    },
+                ),
                 custom: (),
                 node: JoinedStr {
                     values: [
@@ -23,6 +35,12 @@ expression: parse_ast
                                 row: 1,
                                 column: 1,
                             },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
index 380a1fc8..9c93348a 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 18,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 9,
+                    },
+                ),
                 custom: (),
                 node: Constant {
                     value: Str(
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
index 74bf59c3..0af6e48f 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
@@ -8,6 +8,12 @@ expression: parse_ast
             row: 1,
             column: 1,
         },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 18,
+            },
+        ),
         custom: (),
         node: Expr {
             value: Located {
@@ -15,6 +21,12 @@ expression: parse_ast
                     row: 1,
                     column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 10,
+                    },
+                ),
                 custom: (),
                 node: Constant {
                     value: Str(
diff --git a/parser/src/string.rs b/parser/src/string.rs
index 4c8ff851..078ac1e2 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -6,9 +6,12 @@ use crate::{
 };
 use itertools::Itertools;
 
-pub fn parse_strings(values: Vec<(Location, (String, StringKind))>) -> Result<Expr, LexicalError> {
+pub fn parse_strings(
+    values: Vec<(Location, (String, StringKind), Location)>,
+) -> Result<Expr, LexicalError> {
     // Preserve the initial location and kind.
-    let initial_location = values[0].0;
+    let initial_start = values[0].0;
+    let initial_end = values[0].2;
     let initial_kind = (values[0].1 .1 == StringKind::U).then(|| "u".to_owned());
 
     // Determine whether the list of values contains any f-strings. (If not, we can return a
@@ -21,7 +24,8 @@ pub fn parse_strings(values: Vec<(Location, (String, StringKind))>) -> Result<Ex
 
     let take_current = |current: &mut Vec<String>| -> Expr {
         Expr::new(
-            initial_location,
+            initial_start,
+            initial_end,
             ExprKind::Constant {
                 value: Constant::Str(current.drain(..).join("")),
                 kind: initial_kind.clone(),
@@ -29,15 +33,17 @@ pub fn parse_strings(values: Vec<(Location, (String, StringKind))>) -> Result<Ex
         )
     };
 
-    for (location, (string, string_kind)) in values {
+    for (start, (string, string_kind), end) in values {
         match string_kind {
             StringKind::Normal | StringKind::U => current.push(string),
             StringKind::F => {
                 has_fstring = true;
-                for value in parse_located_fstring(&string, location).map_err(|e| LexicalError {
-                    location,
-                    error: LexicalErrorType::FStringError(e.error),
-                })? {
+                for value in
+                    parse_located_fstring(&string, start, end).map_err(|e| LexicalError {
+                        location: start,
+                        error: LexicalErrorType::FStringError(e.error),
+                    })?
+                {
                     match value.node {
                         ExprKind::FormattedValue { .. } => {
                             if !current.is_empty() {
@@ -63,7 +69,11 @@ pub fn parse_strings(values: Vec<(Location, (String, StringKind))>) -> Result<Ex
     }
 
     Ok(if has_fstring {
-        Expr::new(initial_location, ExprKind::JoinedStr { values: deduped })
+        Expr::new(
+            initial_start,
+            initial_end,
+            ExprKind::JoinedStr { values: deduped },
+        )
     } else {
         deduped
             .into_iter()

From 452d28719f246317fd97e6ffb57e26a5a30a7ba7 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Mon, 17 Oct 2022 09:35:03 -0400
Subject: [PATCH 283/459] Expose a method to parse AST from tokens directly

---
 parser/src/parser.rs | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 49d2e53f..9531eb3a 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -5,9 +5,11 @@
 //! parse a whole program, a single statement, or a single
 //! expression.
 
+use std::iter;
+
+use crate::lexer::LexResult;
 pub use crate::mode::Mode;
 use crate::{ast, error::ParseError, lexer, python};
-use std::iter;
 
 /*
  * Parse python code.
@@ -23,6 +25,17 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
     })
 }
 
+/// Parse the token stream for a full python program.
+pub fn parse_program_tokens(
+    lxr: impl IntoIterator<Item=LexResult>,
+    source_path: &str,
+) -> Result<ast::Suite, ParseError> {
+    parse_tokens(lxr, Mode::Module, source_path).map(|top| match top {
+        ast::Mod::Module { body, .. } => body,
+        _ => unreachable!(),
+    })
+}
+
 /// Parses a python expression
 ///
 /// # Example
@@ -80,6 +93,20 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
         .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
 }
 
+// Parse a given token iterator.
+pub fn parse_tokens(
+    lxr: impl IntoIterator<Item=LexResult>,
+    mode: Mode,
+    source_path: &str,
+) -> Result<ast::Mod, ParseError> {
+    let marker_token = (Default::default(), mode.to_marker(), Default::default());
+    let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
+
+    python::TopParser::new()
+        .parse(tokenizer)
+        .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

From efad01d976380d416bf907a068aa9f6bed355f3e Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Mon, 17 Oct 2022 11:59:22 -0400
Subject: [PATCH 284/459] Remove parse_program_tokens

---
 parser/src/parser.rs | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 9531eb3a..90903cf2 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -5,11 +5,10 @@
 //! parse a whole program, a single statement, or a single
 //! expression.
 
-use std::iter;
-
 use crate::lexer::LexResult;
 pub use crate::mode::Mode;
 use crate::{ast, error::ParseError, lexer, python};
+use std::iter;
 
 /*
  * Parse python code.
@@ -25,17 +24,6 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
     })
 }
 
-/// Parse the token stream for a full python program.
-pub fn parse_program_tokens(
-    lxr: impl IntoIterator<Item=LexResult>,
-    source_path: &str,
-) -> Result<ast::Suite, ParseError> {
-    parse_tokens(lxr, Mode::Module, source_path).map(|top| match top {
-        ast::Mod::Module { body, .. } => body,
-        _ => unreachable!(),
-    })
-}
-
 /// Parses a python expression
 ///
 /// # Example
@@ -95,7 +83,7 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
 
 // Parse a given token iterator.
 pub fn parse_tokens(
-    lxr: impl IntoIterator<Item=LexResult>,
+    lxr: impl IntoIterator<Item = LexResult>,
     mode: Mode,
     source_path: &str,
 ) -> Result<ast::Mod, ParseError> {

From 0cd0a0adc87c5a9ef4937edf2e0b087fbddd4577 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Mon, 29 Aug 2022 19:47:02 -0400
Subject: [PATCH 285/459] Add expression context parsing

---
 parser/python.lalrpop                         |  17 +-
 parser/src/context.rs                         | 177 ++++++++++++++++++
 parser/src/lib.rs                             |   1 +
 ...rser__context__tests__ann_assign_name.snap |  48 +++--
 ...python_parser__context__tests__assign.snap |  42 +++++
 ...ser__context__tests__assign_attribute.snap |  86 +++++----
 ...on_parser__context__tests__assign_for.snap |  84 +++++----
 ...n_parser__context__tests__assign_list.snap |  96 ++++++----
 ...ser__context__tests__assign_list_comp.snap | 108 ++++++-----
 ...n_parser__context__tests__assign_name.snap |  72 ++++---
 ...er__context__tests__assign_named_expr.snap |  60 +++---
 ...rser__context__tests__assign_set_comp.snap | 108 ++++++-----
 ...arser__context__tests__assign_starred.snap | 108 ++++++-----
 ...ser__context__tests__assign_subscript.snap |  98 ++++++----
 ..._parser__context__tests__assign_tuple.snap |  96 ++++++----
 ...n_parser__context__tests__assign_with.snap |  48 +++--
 ..._context__tests__aug_assign_attribute.snap |  86 +++++----
 ...rser__context__tests__aug_assign_name.snap |  36 ++--
 ..._context__tests__aug_assign_subscript.snap |  98 ++++++----
 ...parser__context__tests__del_attribute.snap |  38 ++--
 ...thon_parser__context__tests__del_name.snap |  24 ++-
 ...parser__context__tests__del_subscript.snap |  50 ++---
 ...rser__tests__parse_dict_comprehension.snap |   2 +-
 ...ests__parse_double_list_comprehension.snap |   8 +-
 ..._tests__parse_generator_comprehension.snap |   2 +-
 ...parse_if_else_generator_comprehension.snap |   2 +-
 ...rser__tests__parse_list_comprehension.snap |   2 +-
 ...ed_expression_generator_comprehension.snap |   2 +-
 ...n_parser__parser__tests__parse_tuples.snap |   6 +-
 29 files changed, 1024 insertions(+), 581 deletions(-)
 create mode 100644 parser/src/context.rs
 create mode 100644 parser/src/snapshots/rustpython_parser__context__tests__assign.snap

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index cf43f0f2..310790a2 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -8,6 +8,7 @@ use crate::{
     error::{LexicalError, LexicalErrorType},
     function::{ArgumentList, parse_args, parse_params},
     lexer,
+    context::set_context,
     string::parse_strings,
     token::StringKind
 };
@@ -82,7 +83,7 @@ DelStatement: ast::Stmt = {
             location,
             end_location: Some(end_location),
             custom: (),
-            node: ast::StmtKind::Delete { targets },
+            node: ast::StmtKind::Delete { targets: targets.into_iter().map(|expr| set_context(expr, ast::ExprContext::Del)).collect() },
         }
     },
 };
@@ -98,11 +99,11 @@ ExpressionStatement: ast::Stmt = {
                 node: ast::StmtKind::Expr { value: Box::new(expression) }
             }
         } else {
-            let mut targets = vec![expression];
+            let mut targets = vec![set_context(expression, ast::ExprContext::Store)];
             let mut values = suffix;
 
             while values.len() > 1 {
-                targets.push(values.remove(0));
+                targets.push(set_context(values.remove(0), ast::ExprContext::Store));
             }
 
             let value = Box::new(values.into_iter().next().unwrap());
@@ -121,7 +122,7 @@ ExpressionStatement: ast::Stmt = {
             location,
             end_location: Some(end_location),
             node: ast::StmtKind::AugAssign {
-                target: Box::new(target),
+                target: Box::new(set_context(target, ast::ExprContext::Store)),
                 op,
                 value: Box::new(rhs)
             },
@@ -134,7 +135,7 @@ ExpressionStatement: ast::Stmt = {
             location,
             end_location: Some(end_location),
             node: ast::StmtKind::AnnAssign {
-                target: Box::new(target),
+                target: Box::new(set_context(target, ast::ExprContext::Store)),
                 annotation: Box::new(annotation),
                 value: rhs.map(Box::new),
                 simple: if simple { 1 } else { 0 },
@@ -399,7 +400,7 @@ WhileStatement: ast::Stmt = {
 ForStatement: ast::Stmt = {
     <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> <end_location:@R> => {
         let orelse = s2.map(|s| s.2).unwrap_or_default();
-        let target = Box::new(target);
+        let target = Box::new(set_context(target, ast::ExprContext::Store));
         let iter = Box::new(iter);
         let type_comment = None;
         let node = if is_async.is_some() {
@@ -484,7 +485,7 @@ WithStatement: ast::Stmt = {
 
 WithItem: ast::Withitem = {
     <context_expr:Test> <n:("as" Expression)?> => {
-        let optional_vars = n.map(|val| Box::new(val.1));
+        let optional_vars = n.map(|val| Box::new(set_context(val.1, ast::ExprContext::Store)));
         let context_expr = Box::new(context_expr);
         ast::Withitem { context_expr, optional_vars }
     },
@@ -1233,7 +1234,7 @@ SingleForComprehension: ast::Comprehension = {
     <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest> <ifs:ComprehensionIf*> <end_location:@R> => {
         let is_async = is_async.is_some();
         ast::Comprehension {
-            target: Box::new(target),
+            target: Box::new(set_context(target, ast::ExprContext::Store)),
             iter: Box::new(iter),
             ifs,
             is_async: if is_async { 1 } else { 0 },
diff --git a/parser/src/context.rs b/parser/src/context.rs
new file mode 100644
index 00000000..56fdc352
--- /dev/null
+++ b/parser/src/context.rs
@@ -0,0 +1,177 @@
+use rustpython_ast::{Expr, ExprContext, ExprKind};
+
+pub fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
+    match expr.node {
+        ExprKind::Name { id, .. } => Expr {
+            node: ExprKind::Name { id, ctx },
+            ..expr
+        },
+        ExprKind::Tuple { elts, .. } => Expr {
+            node: ExprKind::Tuple {
+                elts: elts
+                    .into_iter()
+                    .map(|elt| set_context(elt, ctx.clone()))
+                    .collect(),
+                ctx,
+            },
+            ..expr
+        },
+        ExprKind::List { elts, .. } => Expr {
+            node: ExprKind::List {
+                elts: elts
+                    .into_iter()
+                    .map(|elt| set_context(elt, ctx.clone()))
+                    .collect(),
+                ctx,
+            },
+            ..expr
+        },
+        ExprKind::Attribute { value, attr, .. } => Expr {
+            node: ExprKind::Attribute { value, attr, ctx },
+            ..expr
+        },
+        ExprKind::Subscript { value, slice, .. } => Expr {
+            node: ExprKind::Subscript { value, slice, ctx },
+            ..expr
+        },
+        ExprKind::Starred { value, .. } => Expr {
+            node: ExprKind::Starred {
+                value: Box::new(set_context(*value, ctx.clone())),
+                ctx,
+            },
+            ..expr
+        },
+        _ => expr,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::parser::parse_program;
+
+    #[test]
+    fn test_assign_name() {
+        let source = String::from("x = (1, 2, 3)");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_assign_tuple() {
+        let source = String::from("(x, y) = (1, 2, 3)");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_assign_list() {
+        let source = String::from("[x, y] = (1, 2, 3)");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_assign_attribute() {
+        let source = String::from("x.y = (1, 2, 3)");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_assign_subscript() {
+        let source = String::from("x[y] = (1, 2, 3)");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_assign_starred() {
+        let source = String::from("(x, *y) = (1, 2, 3)");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_assign_for() {
+        let source = String::from("for x in (1, 2, 3): pass");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_assign_list_comp() {
+        let source = String::from("x = [y for y in (1, 2, 3)]");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_assign_set_comp() {
+        let source = String::from("x = {y for y in (1, 2, 3)}");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_assign_with() {
+        let source = String::from("with 1 as x: pass");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_assign_named_expr() {
+        let source = String::from("if x:= 1: pass");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_ann_assign_name() {
+        let source = String::from("x: int = 1");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_aug_assign_name() {
+        let source = String::from("x += 1");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_aug_assign_attribute() {
+        let source = String::from("x.y += (1, 2, 3)");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_aug_assign_subscript() {
+        let source = String::from("x[y] += (1, 2, 3)");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_del_name() {
+        let source = String::from("del x");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_del_attribute() {
+        let source = String::from("del x.y");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_del_subscript() {
+        let source = String::from("del x[y]");
+        let parse_ast = parse_program(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+}
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index f65f7369..0bd363a8 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -30,5 +30,6 @@ pub mod mode;
 pub mod parser;
 #[rustfmt::skip]
 mod python;
+mod context;
 mod string;
 pub mod token;
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__ann_assign_name.snap b/parser/src/snapshots/rustpython_parser__context__tests__ann_assign_name.snap
index 03b7e437..71888a6f 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__ann_assign_name.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__ann_assign_name.snap
@@ -4,25 +4,29 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 11,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 11,
+            },
+        ),
         custom: (),
         node: AnnAssign {
             target: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 1,
                 },
-                end: Location {
-                    row: 1,
-                    column: 2,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 2,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "x",
@@ -30,14 +34,16 @@ expression: parse_ast
                 },
             },
             annotation: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 4,
                 },
-                end: Location {
-                    row: 1,
-                    column: 7,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 7,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "int",
@@ -46,14 +52,16 @@ expression: parse_ast
             },
             value: Some(
                 Located {
-                    start: Location {
+                    location: Location {
                         row: 1,
                         column: 10,
                     },
-                    end: Location {
-                        row: 1,
-                        column: 11,
-                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 11,
+                        },
+                    ),
                     custom: (),
                     node: Constant {
                         value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign.snap
new file mode 100644
index 00000000..a9fe441d
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign.snap
@@ -0,0 +1,42 @@
+---
+source: compiler/parser/src/context.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 1,
+        },
+        custom: (),
+        node: Assign {
+            targets: [
+                Located {
+                    location: Location {
+                        row: 1,
+                        column: 1,
+                    },
+                    custom: (),
+                    node: Name {
+                        id: "x",
+                        ctx: Store,
+                    },
+                },
+            ],
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 5,
+                },
+                custom: (),
+                node: Constant {
+                    value: Int(
+                        1,
+                    ),
+                    kind: None,
+                },
+            },
+            type_comment: None,
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap
index 9f3abe15..ff527afb 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap
@@ -4,37 +4,43 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 16,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 16,
+            },
+        ),
         custom: (),
         node: Assign {
             targets: [
                 Located {
-                    start: Location {
-                        row: 1,
-                        column: 2,
-                    },
-                    end: Location {
+                    location: Location {
                         row: 1,
-                        column: 4,
+                        column: 1,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 4,
+                        },
+                    ),
                     custom: (),
                     node: Attribute {
                         value: Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 1,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 2,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 2,
+                                },
+                            ),
                             custom: (),
                             node: Name {
                                 id: "x",
@@ -47,26 +53,30 @@ expression: parse_ast
                 },
             ],
             value: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 8,
                 },
-                end: Location {
-                    row: 1,
-                    column: 15,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 15,
+                    },
+                ),
                 custom: (),
                 node: Tuple {
                     elts: [
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 8,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 9,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 9,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -76,14 +86,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 11,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 12,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 12,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -93,14 +105,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 14,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 15,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 15,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap
index 22d92643..e9bb3861 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap
@@ -4,25 +4,29 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 25,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 25,
+            },
+        ),
         custom: (),
         node: For {
             target: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 5,
                 },
-                end: Location {
-                    row: 1,
-                    column: 6,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 6,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "x",
@@ -30,26 +34,30 @@ expression: parse_ast
                 },
             },
             iter: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 11,
                 },
-                end: Location {
-                    row: 1,
-                    column: 18,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 18,
+                    },
+                ),
                 custom: (),
                 node: Tuple {
                     elts: [
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 11,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 12,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 12,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -59,14 +67,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 14,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 15,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 15,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -76,14 +86,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 17,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 18,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 18,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -98,14 +110,16 @@ expression: parse_ast
             },
             body: [
                 Located {
-                    start: Location {
+                    location: Location {
                         row: 1,
                         column: 21,
                     },
-                    end: Location {
-                        row: 1,
-                        column: 25,
-                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 25,
+                        },
+                    ),
                     custom: (),
                     node: Pass,
                 },
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap
index 17abaae4..acedc47f 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap
@@ -4,38 +4,44 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 19,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 19,
+            },
+        ),
         custom: (),
         node: Assign {
             targets: [
                 Located {
-                    start: Location {
+                    location: Location {
                         row: 1,
                         column: 1,
                     },
-                    end: Location {
-                        row: 1,
-                        column: 7,
-                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 7,
+                        },
+                    ),
                     custom: (),
                     node: List {
                         elts: [
                             Located {
-                                start: Location {
+                                location: Location {
                                     row: 1,
                                     column: 2,
                                 },
-                                end: Location {
-                                    row: 1,
-                                    column: 3,
-                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 3,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "x",
@@ -43,14 +49,16 @@ expression: parse_ast
                                 },
                             },
                             Located {
-                                start: Location {
+                                location: Location {
                                     row: 1,
                                     column: 5,
                                 },
-                                end: Location {
-                                    row: 1,
-                                    column: 6,
-                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 6,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "y",
@@ -63,26 +71,30 @@ expression: parse_ast
                 },
             ],
             value: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 11,
                 },
-                end: Location {
-                    row: 1,
-                    column: 18,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 18,
+                    },
+                ),
                 custom: (),
                 node: Tuple {
                     elts: [
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 11,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 12,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 12,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -92,14 +104,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 14,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 15,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 15,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -109,14 +123,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 17,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 18,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 18,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap
index 929f6695..68db2bce 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap
@@ -4,26 +4,30 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 27,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 27,
+            },
+        ),
         custom: (),
         node: Assign {
             targets: [
                 Located {
-                    start: Location {
+                    location: Location {
                         row: 1,
                         column: 1,
                     },
-                    end: Location {
-                        row: 1,
-                        column: 2,
-                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 2,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "x",
@@ -32,25 +36,29 @@ expression: parse_ast
                 },
             ],
             value: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 5,
                 },
-                end: Location {
-                    row: 1,
-                    column: 27,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 27,
+                    },
+                ),
                 custom: (),
                 node: ListComp {
                     elt: Located {
-                        start: Location {
+                        location: Location {
                             row: 1,
                             column: 6,
                         },
-                        end: Location {
-                            row: 1,
-                            column: 7,
-                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 7,
+                            },
+                        ),
                         custom: (),
                         node: Name {
                             id: "y",
@@ -60,14 +68,16 @@ expression: parse_ast
                     generators: [
                         Comprehension {
                             target: Located {
-                                start: Location {
+                                location: Location {
                                     row: 1,
                                     column: 12,
                                 },
-                                end: Location {
-                                    row: 1,
-                                    column: 13,
-                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 13,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "y",
@@ -75,26 +85,30 @@ expression: parse_ast
                                 },
                             },
                             iter: Located {
-                                start: Location {
+                                location: Location {
                                     row: 1,
                                     column: 18,
                                 },
-                                end: Location {
-                                    row: 1,
-                                    column: 25,
-                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 25,
+                                    },
+                                ),
                                 custom: (),
                                 node: Tuple {
                                     elts: [
                                         Located {
-                                            start: Location {
+                                            location: Location {
                                                 row: 1,
                                                 column: 18,
                                             },
-                                            end: Location {
-                                                row: 1,
-                                                column: 19,
-                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 1,
+                                                    column: 19,
+                                                },
+                                            ),
                                             custom: (),
                                             node: Constant {
                                                 value: Int(
@@ -104,14 +118,16 @@ expression: parse_ast
                                             },
                                         },
                                         Located {
-                                            start: Location {
+                                            location: Location {
                                                 row: 1,
                                                 column: 21,
                                             },
-                                            end: Location {
-                                                row: 1,
-                                                column: 22,
-                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 1,
+                                                    column: 22,
+                                                },
+                                            ),
                                             custom: (),
                                             node: Constant {
                                                 value: Int(
@@ -121,14 +137,16 @@ expression: parse_ast
                                             },
                                         },
                                         Located {
-                                            start: Location {
+                                            location: Location {
                                                 row: 1,
                                                 column: 24,
                                             },
-                                            end: Location {
-                                                row: 1,
-                                                column: 25,
-                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 1,
+                                                    column: 25,
+                                                },
+                                            ),
                                             custom: (),
                                             node: Constant {
                                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap
index 614b04ad..7db0871c 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap
@@ -4,26 +4,30 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 14,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 14,
+            },
+        ),
         custom: (),
         node: Assign {
             targets: [
                 Located {
-                    start: Location {
+                    location: Location {
                         row: 1,
                         column: 1,
                     },
-                    end: Location {
-                        row: 1,
-                        column: 2,
-                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 2,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "x",
@@ -32,26 +36,30 @@ expression: parse_ast
                 },
             ],
             value: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 6,
                 },
-                end: Location {
-                    row: 1,
-                    column: 13,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 13,
+                    },
+                ),
                 custom: (),
                 node: Tuple {
                     elts: [
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 6,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 7,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 7,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -61,14 +69,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 9,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 10,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -78,14 +88,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 12,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 13,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap
index 7c9559a8..cc558500 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap
@@ -4,36 +4,42 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 15,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 15,
+            },
+        ),
         custom: (),
         node: If {
             test: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 4,
                 },
-                end: Location {
-                    row: 1,
-                    column: 9,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 9,
+                    },
+                ),
                 custom: (),
                 node: NamedExpr {
                     target: Located {
-                        start: Location {
+                        location: Location {
                             row: 1,
                             column: 4,
                         },
-                        end: Location {
-                            row: 1,
-                            column: 9,
-                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 9,
+                            },
+                        ),
                         custom: (),
                         node: Name {
                             id: "x",
@@ -41,14 +47,16 @@ expression: parse_ast
                         },
                     },
                     value: Located {
-                        start: Location {
+                        location: Location {
                             row: 1,
                             column: 8,
                         },
-                        end: Location {
-                            row: 1,
-                            column: 9,
-                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 9,
+                            },
+                        ),
                         custom: (),
                         node: Constant {
                             value: Int(
@@ -61,14 +69,16 @@ expression: parse_ast
             },
             body: [
                 Located {
-                    start: Location {
+                    location: Location {
                         row: 1,
                         column: 11,
                     },
-                    end: Location {
-                        row: 1,
-                        column: 15,
-                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 15,
+                        },
+                    ),
                     custom: (),
                     node: Pass,
                 },
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap
index abf7db89..a4b587d9 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap
@@ -4,26 +4,30 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 27,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 27,
+            },
+        ),
         custom: (),
         node: Assign {
             targets: [
                 Located {
-                    start: Location {
+                    location: Location {
                         row: 1,
                         column: 1,
                     },
-                    end: Location {
-                        row: 1,
-                        column: 2,
-                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 2,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "x",
@@ -32,25 +36,29 @@ expression: parse_ast
                 },
             ],
             value: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 5,
                 },
-                end: Location {
-                    row: 1,
-                    column: 27,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 27,
+                    },
+                ),
                 custom: (),
                 node: SetComp {
                     elt: Located {
-                        start: Location {
+                        location: Location {
                             row: 1,
                             column: 6,
                         },
-                        end: Location {
-                            row: 1,
-                            column: 7,
-                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 7,
+                            },
+                        ),
                         custom: (),
                         node: Name {
                             id: "y",
@@ -60,14 +68,16 @@ expression: parse_ast
                     generators: [
                         Comprehension {
                             target: Located {
-                                start: Location {
+                                location: Location {
                                     row: 1,
                                     column: 12,
                                 },
-                                end: Location {
-                                    row: 1,
-                                    column: 13,
-                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 13,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "y",
@@ -75,26 +85,30 @@ expression: parse_ast
                                 },
                             },
                             iter: Located {
-                                start: Location {
+                                location: Location {
                                     row: 1,
                                     column: 18,
                                 },
-                                end: Location {
-                                    row: 1,
-                                    column: 25,
-                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 25,
+                                    },
+                                ),
                                 custom: (),
                                 node: Tuple {
                                     elts: [
                                         Located {
-                                            start: Location {
+                                            location: Location {
                                                 row: 1,
                                                 column: 18,
                                             },
-                                            end: Location {
-                                                row: 1,
-                                                column: 19,
-                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 1,
+                                                    column: 19,
+                                                },
+                                            ),
                                             custom: (),
                                             node: Constant {
                                                 value: Int(
@@ -104,14 +118,16 @@ expression: parse_ast
                                             },
                                         },
                                         Located {
-                                            start: Location {
+                                            location: Location {
                                                 row: 1,
                                                 column: 21,
                                             },
-                                            end: Location {
-                                                row: 1,
-                                                column: 22,
-                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 1,
+                                                    column: 22,
+                                                },
+                                            ),
                                             custom: (),
                                             node: Constant {
                                                 value: Int(
@@ -121,14 +137,16 @@ expression: parse_ast
                                             },
                                         },
                                         Located {
-                                            start: Location {
+                                            location: Location {
                                                 row: 1,
                                                 column: 24,
                                             },
-                                            end: Location {
-                                                row: 1,
-                                                column: 25,
-                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 1,
+                                                    column: 25,
+                                                },
+                                            ),
                                             custom: (),
                                             node: Constant {
                                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap
index 24c062e2..775e47bf 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap
@@ -4,38 +4,44 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 20,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 20,
+            },
+        ),
         custom: (),
         node: Assign {
             targets: [
                 Located {
-                    start: Location {
+                    location: Location {
                         row: 1,
                         column: 2,
                     },
-                    end: Location {
-                        row: 1,
-                        column: 7,
-                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 7,
+                        },
+                    ),
                     custom: (),
                     node: Tuple {
                         elts: [
                             Located {
-                                start: Location {
+                                location: Location {
                                     row: 1,
                                     column: 2,
                                 },
-                                end: Location {
-                                    row: 1,
-                                    column: 3,
-                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 3,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "x",
@@ -43,25 +49,29 @@ expression: parse_ast
                                 },
                             },
                             Located {
-                                start: Location {
+                                location: Location {
                                     row: 1,
                                     column: 5,
                                 },
-                                end: Location {
-                                    row: 1,
-                                    column: 7,
-                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 7,
+                                    },
+                                ),
                                 custom: (),
                                 node: Starred {
                                     value: Located {
-                                        start: Location {
+                                        location: Location {
                                             row: 1,
                                             column: 6,
                                         },
-                                        end: Location {
-                                            row: 1,
-                                            column: 7,
-                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 1,
+                                                column: 7,
+                                            },
+                                        ),
                                         custom: (),
                                         node: Name {
                                             id: "y",
@@ -77,26 +87,30 @@ expression: parse_ast
                 },
             ],
             value: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 12,
                 },
-                end: Location {
-                    row: 1,
-                    column: 19,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 19,
+                    },
+                ),
                 custom: (),
                 node: Tuple {
                     elts: [
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 12,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 13,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -106,14 +120,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 15,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 16,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 16,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -123,14 +139,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 18,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 19,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 19,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap
index 7028f2ef..707be61d 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap
@@ -4,37 +4,43 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 17,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 17,
+            },
+        ),
         custom: (),
         node: Assign {
             targets: [
                 Located {
-                    start: Location {
-                        row: 1,
-                        column: 2,
-                    },
-                    end: Location {
+                    location: Location {
                         row: 1,
-                        column: 5,
+                        column: 1,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 5,
+                        },
+                    ),
                     custom: (),
                     node: Subscript {
                         value: Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 1,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 2,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 2,
+                                },
+                            ),
                             custom: (),
                             node: Name {
                                 id: "x",
@@ -42,14 +48,16 @@ expression: parse_ast
                             },
                         },
                         slice: Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 3,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 4,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 4,
+                                },
+                            ),
                             custom: (),
                             node: Name {
                                 id: "y",
@@ -61,26 +69,30 @@ expression: parse_ast
                 },
             ],
             value: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 9,
                 },
-                end: Location {
-                    row: 1,
-                    column: 16,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 16,
+                    },
+                ),
                 custom: (),
                 node: Tuple {
                     elts: [
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 9,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 10,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -90,14 +102,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 12,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 13,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -107,14 +121,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 15,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 16,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 16,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap
index 0d9a1958..08dfed4a 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap
@@ -4,38 +4,44 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 19,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 19,
+            },
+        ),
         custom: (),
         node: Assign {
             targets: [
                 Located {
-                    start: Location {
+                    location: Location {
                         row: 1,
                         column: 2,
                     },
-                    end: Location {
-                        row: 1,
-                        column: 6,
-                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 6,
+                        },
+                    ),
                     custom: (),
                     node: Tuple {
                         elts: [
                             Located {
-                                start: Location {
+                                location: Location {
                                     row: 1,
                                     column: 2,
                                 },
-                                end: Location {
-                                    row: 1,
-                                    column: 3,
-                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 3,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "x",
@@ -43,14 +49,16 @@ expression: parse_ast
                                 },
                             },
                             Located {
-                                start: Location {
+                                location: Location {
                                     row: 1,
                                     column: 5,
                                 },
-                                end: Location {
-                                    row: 1,
-                                    column: 6,
-                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 1,
+                                        column: 6,
+                                    },
+                                ),
                                 custom: (),
                                 node: Name {
                                     id: "y",
@@ -63,26 +71,30 @@ expression: parse_ast
                 },
             ],
             value: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 11,
                 },
-                end: Location {
-                    row: 1,
-                    column: 18,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 18,
+                    },
+                ),
                 custom: (),
                 node: Tuple {
                     elts: [
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 11,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 12,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 12,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -92,14 +104,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 14,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 15,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 15,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -109,14 +123,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 17,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 18,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 18,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_with.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_with.snap
index 042d56b5..88d14411 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_with.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_with.snap
@@ -4,27 +4,31 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 18,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 18,
+            },
+        ),
         custom: (),
         node: With {
             items: [
                 Withitem {
                     context_expr: Located {
-                        start: Location {
+                        location: Location {
                             row: 1,
                             column: 6,
                         },
-                        end: Location {
-                            row: 1,
-                            column: 7,
-                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 7,
+                            },
+                        ),
                         custom: (),
                         node: Constant {
                             value: Int(
@@ -35,14 +39,16 @@ expression: parse_ast
                     },
                     optional_vars: Some(
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 11,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 12,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 12,
+                                },
+                            ),
                             custom: (),
                             node: Name {
                                 id: "x",
@@ -54,14 +60,16 @@ expression: parse_ast
             ],
             body: [
                 Located {
-                    start: Location {
+                    location: Location {
                         row: 1,
                         column: 14,
                     },
-                    end: Location {
-                        row: 1,
-                        column: 18,
-                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 18,
+                        },
+                    ),
                     custom: (),
                     node: Pass,
                 },
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap
index 1e55a560..ea0d8709 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap
@@ -4,36 +4,42 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 17,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 17,
+            },
+        ),
         custom: (),
         node: AugAssign {
             target: Located {
-                start: Location {
-                    row: 1,
-                    column: 2,
-                },
-                end: Location {
+                location: Location {
                     row: 1,
-                    column: 4,
+                    column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 4,
+                    },
+                ),
                 custom: (),
                 node: Attribute {
                     value: Located {
-                        start: Location {
+                        location: Location {
                             row: 1,
                             column: 1,
                         },
-                        end: Location {
-                            row: 1,
-                            column: 2,
-                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 2,
+                            },
+                        ),
                         custom: (),
                         node: Name {
                             id: "x",
@@ -46,26 +52,30 @@ expression: parse_ast
             },
             op: Add,
             value: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 9,
                 },
-                end: Location {
-                    row: 1,
-                    column: 16,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 16,
+                    },
+                ),
                 custom: (),
                 node: Tuple {
                     elts: [
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 9,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 10,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -75,14 +85,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 12,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 13,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -92,14 +104,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 15,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 16,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 16,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_name.snap b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_name.snap
index 576c613f..c11ab23d 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_name.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_name.snap
@@ -4,25 +4,29 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 7,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 7,
+            },
+        ),
         custom: (),
         node: AugAssign {
             target: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 1,
                 },
-                end: Location {
-                    row: 1,
-                    column: 2,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 2,
+                    },
+                ),
                 custom: (),
                 node: Name {
                     id: "x",
@@ -31,14 +35,16 @@ expression: parse_ast
             },
             op: Add,
             value: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 6,
                 },
-                end: Location {
-                    row: 1,
-                    column: 7,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 7,
+                    },
+                ),
                 custom: (),
                 node: Constant {
                     value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap
index 881f665b..0ef667d7 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap
@@ -4,36 +4,42 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 18,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 18,
+            },
+        ),
         custom: (),
         node: AugAssign {
             target: Located {
-                start: Location {
+                location: Location {
                     row: 1,
-                    column: 2,
-                },
-                end: Location {
-                    row: 1,
-                    column: 5,
+                    column: 1,
                 },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 5,
+                    },
+                ),
                 custom: (),
                 node: Subscript {
                     value: Located {
-                        start: Location {
+                        location: Location {
                             row: 1,
                             column: 1,
                         },
-                        end: Location {
-                            row: 1,
-                            column: 2,
-                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 2,
+                            },
+                        ),
                         custom: (),
                         node: Name {
                             id: "x",
@@ -41,14 +47,16 @@ expression: parse_ast
                         },
                     },
                     slice: Located {
-                        start: Location {
+                        location: Location {
                             row: 1,
                             column: 3,
                         },
-                        end: Location {
-                            row: 1,
-                            column: 4,
-                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 4,
+                            },
+                        ),
                         custom: (),
                         node: Name {
                             id: "y",
@@ -60,26 +68,30 @@ expression: parse_ast
             },
             op: Add,
             value: Located {
-                start: Location {
+                location: Location {
                     row: 1,
                     column: 10,
                 },
-                end: Location {
-                    row: 1,
-                    column: 17,
-                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 17,
+                    },
+                ),
                 custom: (),
                 node: Tuple {
                     elts: [
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 10,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 11,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 11,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -89,14 +101,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 13,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 14,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 14,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
@@ -106,14 +120,16 @@ expression: parse_ast
                             },
                         },
                         Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 16,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 17,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 17,
+                                },
+                            ),
                             custom: (),
                             node: Constant {
                                 value: Int(
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__del_attribute.snap b/parser/src/snapshots/rustpython_parser__context__tests__del_attribute.snap
index 39e650e2..21de28d7 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__del_attribute.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__del_attribute.snap
@@ -4,37 +4,43 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 8,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 8,
+            },
+        ),
         custom: (),
         node: Delete {
             targets: [
                 Located {
-                    start: Location {
-                        row: 1,
-                        column: 6,
-                    },
-                    end: Location {
+                    location: Location {
                         row: 1,
-                        column: 8,
+                        column: 5,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 8,
+                        },
+                    ),
                     custom: (),
                     node: Attribute {
                         value: Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 5,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 6,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 6,
+                                },
+                            ),
                             custom: (),
                             node: Name {
                                 id: "x",
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__del_name.snap b/parser/src/snapshots/rustpython_parser__context__tests__del_name.snap
index 396a5700..5d8d8e89 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__del_name.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__del_name.snap
@@ -4,26 +4,30 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 6,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 6,
+            },
+        ),
         custom: (),
         node: Delete {
             targets: [
                 Located {
-                    start: Location {
+                    location: Location {
                         row: 1,
                         column: 5,
                     },
-                    end: Location {
-                        row: 1,
-                        column: 6,
-                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 6,
+                        },
+                    ),
                     custom: (),
                     node: Name {
                         id: "x",
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__del_subscript.snap b/parser/src/snapshots/rustpython_parser__context__tests__del_subscript.snap
index 56da6c49..7a85274e 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__del_subscript.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__del_subscript.snap
@@ -4,37 +4,43 @@ expression: parse_ast
 ---
 [
     Located {
-        start: Location {
+        location: Location {
             row: 1,
             column: 1,
         },
-        end: Location {
-            row: 1,
-            column: 9,
-        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 9,
+            },
+        ),
         custom: (),
         node: Delete {
             targets: [
                 Located {
-                    start: Location {
+                    location: Location {
                         row: 1,
-                        column: 6,
-                    },
-                    end: Location {
-                        row: 1,
-                        column: 9,
+                        column: 5,
                     },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 9,
+                        },
+                    ),
                     custom: (),
                     node: Subscript {
                         value: Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 5,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 6,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 6,
+                                },
+                            ),
                             custom: (),
                             node: Name {
                                 id: "x",
@@ -42,14 +48,16 @@ expression: parse_ast
                             },
                         },
                         slice: Located {
-                            start: Location {
+                            location: Location {
                                 row: 1,
                                 column: 7,
                             },
-                            end: Location {
-                                row: 1,
-                                column: 8,
-                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 8,
+                                },
+                            ),
                             custom: (),
                             node: Name {
                                 id: "y",
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap
index a75ccdce..7d5a69fa 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap
@@ -65,7 +65,7 @@ Located {
                     custom: (),
                     node: Name {
                         id: "y",
-                        ctx: Load,
+                        ctx: Store,
                     },
                 },
                 iter: Located {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
index 3018e3ff..9cc392c8 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
@@ -62,7 +62,7 @@ Located {
                                 custom: (),
                                 node: Name {
                                     id: "y",
-                                    ctx: Load,
+                                    ctx: Store,
                                 },
                             },
                             Located {
@@ -79,11 +79,11 @@ Located {
                                 custom: (),
                                 node: Name {
                                     id: "y2",
-                                    ctx: Load,
+                                    ctx: Store,
                                 },
                             },
                         ],
-                        ctx: Load,
+                        ctx: Store,
                     },
                 },
                 iter: Located {
@@ -121,7 +121,7 @@ Located {
                     custom: (),
                     node: Name {
                         id: "a",
-                        ctx: Load,
+                        ctx: Store,
                     },
                 },
                 iter: Located {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap
index 0b85a1ab..1ceab196 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap
@@ -48,7 +48,7 @@ Located {
                     custom: (),
                     node: Name {
                         id: "y",
-                        ctx: Load,
+                        ctx: Store,
                     },
                 },
                 iter: Located {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
index 4505d92c..6f8b7c27 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
@@ -97,7 +97,7 @@ Located {
                     custom: (),
                     node: Name {
                         id: "y",
-                        ctx: Load,
+                        ctx: Store,
                     },
                 },
                 iter: Located {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap
index d6916653..f932e7db 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap
@@ -48,7 +48,7 @@ Located {
                     custom: (),
                     node: Name {
                         id: "y",
-                        ctx: Load,
+                        ctx: Store,
                     },
                 },
                 iter: Located {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
index 660f0fac..2f534dd8 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
@@ -115,7 +115,7 @@ Located {
                     custom: (),
                     node: Name {
                         id: "y",
-                        ctx: Load,
+                        ctx: Store,
                     },
                 },
                 iter: Located {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap
index d50b57d7..c8e252c4 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap
@@ -45,7 +45,7 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                 custom: (),
                                 node: Name {
                                     id: "a",
-                                    ctx: Load,
+                                    ctx: Store,
                                 },
                             },
                             Located {
@@ -62,11 +62,11 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                 custom: (),
                                 node: Name {
                                     id: "b",
-                                    ctx: Load,
+                                    ctx: Store,
                                 },
                             },
                         ],
-                        ctx: Load,
+                        ctx: Store,
                     },
                 },
             ],

From a96bc6ad3e191d4fcffdbdd6e4a3bd0383095994 Mon Sep 17 00:00:00 2001
From: Kuan-Chun Wang <jeff1jeffo@gmail.com>
Date: Fri, 21 Oct 2022 12:06:50 +0800
Subject: [PATCH 286/459] Fix `global` behavior on class (#4234)

---
 codegen/src/compile.rs | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 9475b8c7..6666d231 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -1275,6 +1275,15 @@ impl Compiler {
 
         let prev_class_name = std::mem::replace(&mut self.class_name, Some(name.to_owned()));
 
+        // Check if the class is declared global
+        let symbol_table = self.symbol_table_stack.last().unwrap();
+        let symbol = symbol_table.lookup(name.as_ref()).expect(
+            "The symbol must be present in the symbol table, even when it is undefined in python.",
+        );
+        let mut global_path_prefix = Vec::new();
+        if symbol.scope == SymbolScope::GlobalExplicit {
+            global_path_prefix.append(&mut self.qualified_path);
+        }
         self.push_qualified_path(name);
         let qualified_name = self.qualified_path.join(".");
 
@@ -1287,7 +1296,7 @@ impl Compiler {
         let dunder_module = self.name("__module__");
         self.emit(Instruction::StoreLocal(dunder_module));
         self.emit_constant(ConstantData::Str {
-            value: qualified_name.clone(),
+            value: qualified_name,
         });
         let qualname = self.name("__qualname__");
         self.emit(Instruction::StoreLocal(qualname));
@@ -1323,6 +1332,7 @@ impl Compiler {
 
         self.class_name = prev_class_name;
         self.qualified_path.pop();
+        self.qualified_path.append(global_path_prefix.as_mut());
         self.ctx = prev_ctx;
 
         let mut funcflags = bytecode::MakeFunctionFlags::empty();
@@ -1342,7 +1352,7 @@ impl Compiler {
         self.emit(Instruction::MakeFunction(funcflags));
 
         self.emit_constant(ConstantData::Str {
-            value: qualified_name,
+            value: name.to_owned(),
         });
 
         let call = self.compile_call_inner(2, bases, keywords)?;

From ac77f7f501a5ca289363f618ec63fe3872bf9b80 Mon Sep 17 00:00:00 2001
From: dvermd <315743+dvermd@users.noreply.github.com>
Date: Mon, 17 Oct 2022 19:09:15 +0200
Subject: [PATCH 287/459] Fix ast types' _fields and use 0-based column

---
 ast/asdl_rs.py                                |  2 +-
 core/src/location.rs                          |  2 +-
 parser/src/parser.rs                          | 12 ++--
 ...rser__context__tests__ann_assign_name.snap | 16 +++---
 ...ser__context__tests__assign_attribute.snap | 28 +++++-----
 ...on_parser__context__tests__assign_for.snap | 28 +++++-----
 ...n_parser__context__tests__assign_list.snap | 32 +++++------
 ...ser__context__tests__assign_list_comp.snap | 36 ++++++------
 ...n_parser__context__tests__assign_name.snap | 24 ++++----
 ...er__context__tests__assign_named_expr.snap | 20 +++----
 ...rser__context__tests__assign_set_comp.snap | 36 ++++++------
 ...arser__context__tests__assign_starred.snap | 36 ++++++------
 ...ser__context__tests__assign_subscript.snap | 32 +++++------
 ..._parser__context__tests__assign_tuple.snap | 32 +++++------
 ...n_parser__context__tests__assign_with.snap | 16 +++---
 ..._context__tests__aug_assign_attribute.snap | 28 +++++-----
 ...rser__context__tests__aug_assign_name.snap | 12 ++--
 ..._context__tests__aug_assign_subscript.snap | 32 +++++------
 ...parser__context__tests__del_attribute.snap | 12 ++--
 ...thon_parser__context__tests__del_name.snap |  8 +--
 ...parser__context__tests__del_subscript.snap | 16 +++---
 ...s__fstring_parse_selfdocumenting_base.snap |  4 +-
 ...tring_parse_selfdocumenting_base_more.snap |  8 +--
 ..._fstring_parse_selfdocumenting_format.snap |  4 +-
 ...parser__fstring__tests__parse_fstring.snap |  8 +--
 ..._fstring__tests__parse_fstring_equals.snap | 12 ++--
 ...ing__tests__parse_fstring_nested_spec.snap |  8 +--
 ...ring__tests__parse_fstring_not_equals.snap | 12 ++--
 ..._tests__parse_fstring_not_nested_spec.snap |  4 +-
 ...sts__parse_fstring_selfdoc_prec_space.snap |  4 +-
 ..._parse_fstring_selfdoc_trailing_space.snap |  4 +-
 ...ring__tests__parse_fstring_yield_expr.snap |  4 +-
 ...on_parser__parser__tests__parse_class.snap | 10 ++--
 ...rser__tests__parse_dict_comprehension.snap | 20 +++----
 ...ests__parse_double_list_comprehension.snap | 56 +++++++++----------
 ...parser__parser__tests__parse_f_string.snap | 12 ++--
 ..._tests__parse_generator_comprehension.snap | 16 +++---
 ...er__parser__tests__parse_if_elif_else.snap | 14 ++---
 ...parse_if_else_generator_comprehension.snap | 28 +++++-----
 ...n_parser__parser__tests__parse_kwargs.snap | 24 ++++----
 ...n_parser__parser__tests__parse_lambda.snap | 28 +++++-----
 ...rser__tests__parse_list_comprehension.snap | 16 +++---
 ...ed_expression_generator_comprehension.snap | 32 +++++------
 ..._parser__parser__tests__parse_print_2.snap | 20 +++----
 ...ser__parser__tests__parse_print_hello.snap | 16 +++---
 ...n_parser__parser__tests__parse_string.snap |  8 +--
 ...n_parser__parser__tests__parse_tuples.snap | 28 +++++-----
 ...tring__tests__parse_f_string_concat_1.snap | 12 ++--
 ...tring__tests__parse_f_string_concat_2.snap | 12 ++--
 ...tring__tests__parse_f_string_concat_3.snap | 20 +++----
 ...r__string__tests__parse_string_concat.snap |  8 +--
 ..._parse_string_triple_quotes_with_kind.snap |  8 +--
 ...ing__tests__parse_u_f_string_concat_1.snap | 12 ++--
 ...ing__tests__parse_u_f_string_concat_2.snap | 12 ++--
 ...tring__tests__parse_u_string_concat_1.snap |  8 +--
 ...tring__tests__parse_u_string_concat_2.snap |  8 +--
 56 files changed, 480 insertions(+), 480 deletions(-)

diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py
index f0617055..06ee2e57 100755
--- a/ast/asdl_rs.py
+++ b/ast/asdl_rs.py
@@ -449,7 +449,7 @@ def gen_classdef(self, name, fields, attrs, depth, base="AstNode"):
             f"ctx.new_str(ascii!({json.dumps(f.name)})).into()" for f in fields
         )
         self.emit(
-            f"class.set_attr(identifier!(ctx, _fields), ctx.new_list(vec![{fields}]).into());",
+            f"class.set_attr(identifier!(ctx, _fields), ctx.new_tuple(vec![{fields}]).into());",
             depth + 2,
         )
         attrs = ",".join(
diff --git a/core/src/location.rs b/core/src/location.rs
index 73d6ec89..c4b7aa1f 100644
--- a/core/src/location.rs
+++ b/core/src/location.rs
@@ -43,7 +43,7 @@ impl Location {
 
     pub fn reset(&mut self) {
         self.row = 1;
-        self.column = 1;
+        self.column = 0;
     }
 
     pub fn go_right(&mut self) {
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 90903cf2..9bab28cf 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -35,13 +35,13 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
 /// assert_eq!(
 ///     expr,
 ///     ast::Expr {
-///         location: ast::Location::new(1, 3),
-///         end_location: Some(ast::Location::new(1, 6)),
+///         location: ast::Location::new(1, 2),
+///         end_location: Some(ast::Location::new(1, 5)),
 ///         custom: (),
 ///         node: ast::ExprKind::BinOp {
 ///             left: Box::new(ast::Expr {
-///                 location: ast::Location::new(1, 1),
-///                 end_location: Some(ast::Location::new(1, 2)),
+///                 location: ast::Location::new(1, 0),
+///                 end_location: Some(ast::Location::new(1, 1)),
 ///                 custom: (),
 ///                 node: ast::ExprKind::Constant {
 ///                     value: ast::Constant::Int(1.into()),
@@ -50,8 +50,8 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
 ///             }),
 ///             op: ast::Operator::Add,
 ///             right: Box::new(ast::Expr {
-///                 location: ast::Location::new(1, 5),
-///                 end_location: Some(ast::Location::new(1, 6)),
+///                 location: ast::Location::new(1, 4),
+///                 end_location: Some(ast::Location::new(1, 5)),
 ///                 custom: (),
 ///                 node: ast::ExprKind::Constant {
 ///                     value: ast::Constant::Int(2.into()),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__ann_assign_name.snap b/parser/src/snapshots/rustpython_parser__context__tests__ann_assign_name.snap
index 71888a6f..78ccf768 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__ann_assign_name.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__ann_assign_name.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 11,
+                column: 10,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             target: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 2,
+                        column: 1,
                     },
                 ),
                 custom: (),
@@ -36,12 +36,12 @@ expression: parse_ast
             annotation: Located {
                 location: Location {
                     row: 1,
-                    column: 4,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 7,
+                        column: 6,
                     },
                 ),
                 custom: (),
@@ -54,12 +54,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 10,
+                        column: 9,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 11,
+                            column: 10,
                         },
                     ),
                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap
index ff527afb..a247b566 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 16,
+                column: 15,
             },
         ),
         custom: (),
@@ -20,12 +20,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 1,
+                        column: 0,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 4,
+                            column: 3,
                         },
                     ),
                     custom: (),
@@ -33,12 +33,12 @@ expression: parse_ast
                         value: Located {
                             location: Location {
                                 row: 1,
-                                column: 1,
+                                column: 0,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 2,
+                                    column: 1,
                                 },
                             ),
                             custom: (),
@@ -55,12 +55,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 8,
+                    column: 7,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 15,
+                        column: 14,
                     },
                 ),
                 custom: (),
@@ -69,12 +69,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 8,
+                                column: 7,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 9,
+                                    column: 8,
                                 },
                             ),
                             custom: (),
@@ -88,12 +88,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 11,
+                                column: 10,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 12,
+                                    column: 11,
                                 },
                             ),
                             custom: (),
@@ -107,12 +107,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 14,
+                                column: 13,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 15,
+                                    column: 14,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap
index e9bb3861..bbacec76 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 25,
+                column: 24,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             target: Located {
                 location: Location {
                     row: 1,
-                    column: 5,
+                    column: 4,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 6,
+                        column: 5,
                     },
                 ),
                 custom: (),
@@ -36,12 +36,12 @@ expression: parse_ast
             iter: Located {
                 location: Location {
                     row: 1,
-                    column: 11,
+                    column: 10,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 18,
+                        column: 17,
                     },
                 ),
                 custom: (),
@@ -50,12 +50,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 11,
+                                column: 10,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 12,
+                                    column: 11,
                                 },
                             ),
                             custom: (),
@@ -69,12 +69,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 14,
+                                column: 13,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 15,
+                                    column: 14,
                                 },
                             ),
                             custom: (),
@@ -88,12 +88,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 17,
+                                column: 16,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 18,
+                                    column: 17,
                                 },
                             ),
                             custom: (),
@@ -112,12 +112,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 21,
+                        column: 20,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 25,
+                            column: 24,
                         },
                     ),
                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap
index acedc47f..c90e5a35 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 19,
+                column: 18,
             },
         ),
         custom: (),
@@ -20,12 +20,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 1,
+                        column: 0,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 7,
+                            column: 6,
                         },
                     ),
                     custom: (),
@@ -34,12 +34,12 @@ expression: parse_ast
                             Located {
                                 location: Location {
                                     row: 1,
-                                    column: 2,
+                                    column: 1,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 3,
+                                        column: 2,
                                     },
                                 ),
                                 custom: (),
@@ -51,12 +51,12 @@ expression: parse_ast
                             Located {
                                 location: Location {
                                     row: 1,
-                                    column: 5,
+                                    column: 4,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 6,
+                                        column: 5,
                                     },
                                 ),
                                 custom: (),
@@ -73,12 +73,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 11,
+                    column: 10,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 18,
+                        column: 17,
                     },
                 ),
                 custom: (),
@@ -87,12 +87,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 11,
+                                column: 10,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 12,
+                                    column: 11,
                                 },
                             ),
                             custom: (),
@@ -106,12 +106,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 14,
+                                column: 13,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 15,
+                                    column: 14,
                                 },
                             ),
                             custom: (),
@@ -125,12 +125,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 17,
+                                column: 16,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 18,
+                                    column: 17,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap
index 68db2bce..9b6d6ac9 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 27,
+                column: 26,
             },
         ),
         custom: (),
@@ -20,12 +20,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 1,
+                        column: 0,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 2,
+                            column: 1,
                         },
                     ),
                     custom: (),
@@ -38,12 +38,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 5,
+                    column: 4,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 27,
+                        column: 26,
                     },
                 ),
                 custom: (),
@@ -51,12 +51,12 @@ expression: parse_ast
                     elt: Located {
                         location: Location {
                             row: 1,
-                            column: 6,
+                            column: 5,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 7,
+                                column: 6,
                             },
                         ),
                         custom: (),
@@ -70,12 +70,12 @@ expression: parse_ast
                             target: Located {
                                 location: Location {
                                     row: 1,
-                                    column: 12,
+                                    column: 11,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 13,
+                                        column: 12,
                                     },
                                 ),
                                 custom: (),
@@ -87,12 +87,12 @@ expression: parse_ast
                             iter: Located {
                                 location: Location {
                                     row: 1,
-                                    column: 18,
+                                    column: 17,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 25,
+                                        column: 24,
                                     },
                                 ),
                                 custom: (),
@@ -101,12 +101,12 @@ expression: parse_ast
                                         Located {
                                             location: Location {
                                                 row: 1,
-                                                column: 18,
+                                                column: 17,
                                             },
                                             end_location: Some(
                                                 Location {
                                                     row: 1,
-                                                    column: 19,
+                                                    column: 18,
                                                 },
                                             ),
                                             custom: (),
@@ -120,12 +120,12 @@ expression: parse_ast
                                         Located {
                                             location: Location {
                                                 row: 1,
-                                                column: 21,
+                                                column: 20,
                                             },
                                             end_location: Some(
                                                 Location {
                                                     row: 1,
-                                                    column: 22,
+                                                    column: 21,
                                                 },
                                             ),
                                             custom: (),
@@ -139,12 +139,12 @@ expression: parse_ast
                                         Located {
                                             location: Location {
                                                 row: 1,
-                                                column: 24,
+                                                column: 23,
                                             },
                                             end_location: Some(
                                                 Location {
                                                     row: 1,
-                                                    column: 25,
+                                                    column: 24,
                                                 },
                                             ),
                                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap
index 7db0871c..e0cdb858 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 14,
+                column: 13,
             },
         ),
         custom: (),
@@ -20,12 +20,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 1,
+                        column: 0,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 2,
+                            column: 1,
                         },
                     ),
                     custom: (),
@@ -38,12 +38,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 6,
+                    column: 5,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 13,
+                        column: 12,
                     },
                 ),
                 custom: (),
@@ -52,12 +52,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 6,
+                                column: 5,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 7,
+                                    column: 6,
                                 },
                             ),
                             custom: (),
@@ -71,12 +71,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 9,
+                                column: 8,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 10,
+                                    column: 9,
                                 },
                             ),
                             custom: (),
@@ -90,12 +90,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 12,
+                                column: 11,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 13,
+                                    column: 12,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap
index cc558500..68d212a4 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 15,
+                column: 14,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             test: Located {
                 location: Location {
                     row: 1,
-                    column: 4,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 9,
+                        column: 8,
                     },
                 ),
                 custom: (),
@@ -32,12 +32,12 @@ expression: parse_ast
                     target: Located {
                         location: Location {
                             row: 1,
-                            column: 4,
+                            column: 3,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 9,
+                                column: 8,
                             },
                         ),
                         custom: (),
@@ -49,12 +49,12 @@ expression: parse_ast
                     value: Located {
                         location: Location {
                             row: 1,
-                            column: 8,
+                            column: 7,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 9,
+                                column: 8,
                             },
                         ),
                         custom: (),
@@ -71,12 +71,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 11,
+                        column: 10,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 15,
+                            column: 14,
                         },
                     ),
                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap
index a4b587d9..84dba820 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 27,
+                column: 26,
             },
         ),
         custom: (),
@@ -20,12 +20,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 1,
+                        column: 0,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 2,
+                            column: 1,
                         },
                     ),
                     custom: (),
@@ -38,12 +38,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 5,
+                    column: 4,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 27,
+                        column: 26,
                     },
                 ),
                 custom: (),
@@ -51,12 +51,12 @@ expression: parse_ast
                     elt: Located {
                         location: Location {
                             row: 1,
-                            column: 6,
+                            column: 5,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 7,
+                                column: 6,
                             },
                         ),
                         custom: (),
@@ -70,12 +70,12 @@ expression: parse_ast
                             target: Located {
                                 location: Location {
                                     row: 1,
-                                    column: 12,
+                                    column: 11,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 13,
+                                        column: 12,
                                     },
                                 ),
                                 custom: (),
@@ -87,12 +87,12 @@ expression: parse_ast
                             iter: Located {
                                 location: Location {
                                     row: 1,
-                                    column: 18,
+                                    column: 17,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 25,
+                                        column: 24,
                                     },
                                 ),
                                 custom: (),
@@ -101,12 +101,12 @@ expression: parse_ast
                                         Located {
                                             location: Location {
                                                 row: 1,
-                                                column: 18,
+                                                column: 17,
                                             },
                                             end_location: Some(
                                                 Location {
                                                     row: 1,
-                                                    column: 19,
+                                                    column: 18,
                                                 },
                                             ),
                                             custom: (),
@@ -120,12 +120,12 @@ expression: parse_ast
                                         Located {
                                             location: Location {
                                                 row: 1,
-                                                column: 21,
+                                                column: 20,
                                             },
                                             end_location: Some(
                                                 Location {
                                                     row: 1,
-                                                    column: 22,
+                                                    column: 21,
                                                 },
                                             ),
                                             custom: (),
@@ -139,12 +139,12 @@ expression: parse_ast
                                         Located {
                                             location: Location {
                                                 row: 1,
-                                                column: 24,
+                                                column: 23,
                                             },
                                             end_location: Some(
                                                 Location {
                                                     row: 1,
-                                                    column: 25,
+                                                    column: 24,
                                                 },
                                             ),
                                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap
index 775e47bf..2e4256e2 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 20,
+                column: 19,
             },
         ),
         custom: (),
@@ -20,12 +20,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 2,
+                        column: 1,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 7,
+                            column: 6,
                         },
                     ),
                     custom: (),
@@ -34,12 +34,12 @@ expression: parse_ast
                             Located {
                                 location: Location {
                                     row: 1,
-                                    column: 2,
+                                    column: 1,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 3,
+                                        column: 2,
                                     },
                                 ),
                                 custom: (),
@@ -51,12 +51,12 @@ expression: parse_ast
                             Located {
                                 location: Location {
                                     row: 1,
-                                    column: 5,
+                                    column: 4,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 7,
+                                        column: 6,
                                     },
                                 ),
                                 custom: (),
@@ -64,12 +64,12 @@ expression: parse_ast
                                     value: Located {
                                         location: Location {
                                             row: 1,
-                                            column: 6,
+                                            column: 5,
                                         },
                                         end_location: Some(
                                             Location {
                                                 row: 1,
-                                                column: 7,
+                                                column: 6,
                                             },
                                         ),
                                         custom: (),
@@ -89,12 +89,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 12,
+                    column: 11,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 19,
+                        column: 18,
                     },
                 ),
                 custom: (),
@@ -103,12 +103,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 12,
+                                column: 11,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 13,
+                                    column: 12,
                                 },
                             ),
                             custom: (),
@@ -122,12 +122,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 15,
+                                column: 14,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 16,
+                                    column: 15,
                                 },
                             ),
                             custom: (),
@@ -141,12 +141,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 18,
+                                column: 17,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 19,
+                                    column: 18,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap
index 707be61d..28420ebc 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 17,
+                column: 16,
             },
         ),
         custom: (),
@@ -20,12 +20,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 1,
+                        column: 0,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 5,
+                            column: 4,
                         },
                     ),
                     custom: (),
@@ -33,12 +33,12 @@ expression: parse_ast
                         value: Located {
                             location: Location {
                                 row: 1,
-                                column: 1,
+                                column: 0,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 2,
+                                    column: 1,
                                 },
                             ),
                             custom: (),
@@ -50,12 +50,12 @@ expression: parse_ast
                         slice: Located {
                             location: Location {
                                 row: 1,
-                                column: 3,
+                                column: 2,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 4,
+                                    column: 3,
                                 },
                             ),
                             custom: (),
@@ -71,12 +71,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 9,
+                    column: 8,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 16,
+                        column: 15,
                     },
                 ),
                 custom: (),
@@ -85,12 +85,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 9,
+                                column: 8,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 10,
+                                    column: 9,
                                 },
                             ),
                             custom: (),
@@ -104,12 +104,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 12,
+                                column: 11,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 13,
+                                    column: 12,
                                 },
                             ),
                             custom: (),
@@ -123,12 +123,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 15,
+                                column: 14,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 16,
+                                    column: 15,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap
index 08dfed4a..30a51df8 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 19,
+                column: 18,
             },
         ),
         custom: (),
@@ -20,12 +20,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 2,
+                        column: 1,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 6,
+                            column: 5,
                         },
                     ),
                     custom: (),
@@ -34,12 +34,12 @@ expression: parse_ast
                             Located {
                                 location: Location {
                                     row: 1,
-                                    column: 2,
+                                    column: 1,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 3,
+                                        column: 2,
                                     },
                                 ),
                                 custom: (),
@@ -51,12 +51,12 @@ expression: parse_ast
                             Located {
                                 location: Location {
                                     row: 1,
-                                    column: 5,
+                                    column: 4,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 6,
+                                        column: 5,
                                     },
                                 ),
                                 custom: (),
@@ -73,12 +73,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 11,
+                    column: 10,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 18,
+                        column: 17,
                     },
                 ),
                 custom: (),
@@ -87,12 +87,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 11,
+                                column: 10,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 12,
+                                    column: 11,
                                 },
                             ),
                             custom: (),
@@ -106,12 +106,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 14,
+                                column: 13,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 15,
+                                    column: 14,
                                 },
                             ),
                             custom: (),
@@ -125,12 +125,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 17,
+                                column: 16,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 18,
+                                    column: 17,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_with.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_with.snap
index 88d14411..100b6728 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_with.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_with.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 18,
+                column: 17,
             },
         ),
         custom: (),
@@ -21,12 +21,12 @@ expression: parse_ast
                     context_expr: Located {
                         location: Location {
                             row: 1,
-                            column: 6,
+                            column: 5,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 7,
+                                column: 6,
                             },
                         ),
                         custom: (),
@@ -41,12 +41,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 11,
+                                column: 10,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 12,
+                                    column: 11,
                                 },
                             ),
                             custom: (),
@@ -62,12 +62,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 14,
+                        column: 13,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 18,
+                            column: 17,
                         },
                     ),
                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap
index ea0d8709..3ba9d045 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 17,
+                column: 16,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             target: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 4,
+                        column: 3,
                     },
                 ),
                 custom: (),
@@ -32,12 +32,12 @@ expression: parse_ast
                     value: Located {
                         location: Location {
                             row: 1,
-                            column: 1,
+                            column: 0,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 2,
+                                column: 1,
                             },
                         ),
                         custom: (),
@@ -54,12 +54,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 9,
+                    column: 8,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 16,
+                        column: 15,
                     },
                 ),
                 custom: (),
@@ -68,12 +68,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 9,
+                                column: 8,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 10,
+                                    column: 9,
                                 },
                             ),
                             custom: (),
@@ -87,12 +87,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 12,
+                                column: 11,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 13,
+                                    column: 12,
                                 },
                             ),
                             custom: (),
@@ -106,12 +106,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 15,
+                                column: 14,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 16,
+                                    column: 15,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_name.snap b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_name.snap
index c11ab23d..0035310f 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_name.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_name.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 7,
+                column: 6,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             target: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 2,
+                        column: 1,
                     },
                 ),
                 custom: (),
@@ -37,12 +37,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 6,
+                    column: 5,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 7,
+                        column: 6,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap
index 0ef667d7..ebb262a3 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 18,
+                column: 17,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             target: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 5,
+                        column: 4,
                     },
                 ),
                 custom: (),
@@ -32,12 +32,12 @@ expression: parse_ast
                     value: Located {
                         location: Location {
                             row: 1,
-                            column: 1,
+                            column: 0,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 2,
+                                column: 1,
                             },
                         ),
                         custom: (),
@@ -49,12 +49,12 @@ expression: parse_ast
                     slice: Located {
                         location: Location {
                             row: 1,
-                            column: 3,
+                            column: 2,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 4,
+                                column: 3,
                             },
                         ),
                         custom: (),
@@ -70,12 +70,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 10,
+                    column: 9,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 17,
+                        column: 16,
                     },
                 ),
                 custom: (),
@@ -84,12 +84,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 10,
+                                column: 9,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 11,
+                                    column: 10,
                                 },
                             ),
                             custom: (),
@@ -103,12 +103,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 13,
+                                column: 12,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 14,
+                                    column: 13,
                                 },
                             ),
                             custom: (),
@@ -122,12 +122,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 16,
+                                column: 15,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 17,
+                                    column: 16,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__del_attribute.snap b/parser/src/snapshots/rustpython_parser__context__tests__del_attribute.snap
index 21de28d7..f44c23b5 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__del_attribute.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__del_attribute.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 8,
+                column: 7,
             },
         ),
         custom: (),
@@ -20,12 +20,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 5,
+                        column: 4,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 8,
+                            column: 7,
                         },
                     ),
                     custom: (),
@@ -33,12 +33,12 @@ expression: parse_ast
                         value: Located {
                             location: Location {
                                 row: 1,
-                                column: 5,
+                                column: 4,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 6,
+                                    column: 5,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__del_name.snap b/parser/src/snapshots/rustpython_parser__context__tests__del_name.snap
index 5d8d8e89..56208d20 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__del_name.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__del_name.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 6,
+                column: 5,
             },
         ),
         custom: (),
@@ -20,12 +20,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 5,
+                        column: 4,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 6,
+                            column: 5,
                         },
                     ),
                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__del_subscript.snap b/parser/src/snapshots/rustpython_parser__context__tests__del_subscript.snap
index 7a85274e..dca2e7ce 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__del_subscript.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__del_subscript.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 9,
+                column: 8,
             },
         ),
         custom: (),
@@ -20,12 +20,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 5,
+                        column: 4,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 9,
+                            column: 8,
                         },
                     ),
                     custom: (),
@@ -33,12 +33,12 @@ expression: parse_ast
                         value: Located {
                             location: Location {
                                 row: 1,
-                                column: 5,
+                                column: 4,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 6,
+                                    column: 5,
                                 },
                             ),
                             custom: (),
@@ -50,12 +50,12 @@ expression: parse_ast
                         slice: Located {
                             location: Location {
                                 row: 1,
-                                column: 7,
+                                column: 6,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 8,
+                                    column: 7,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
index d54af22b..0941873f 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
@@ -57,12 +57,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 6,
+                        column: 5,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
index df3a617e..92a05f21 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
@@ -76,12 +76,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 6,
+                        column: 5,
                     },
                 ),
                 custom: (),
@@ -167,12 +167,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 8,
+                        column: 7,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
index a224dd43..ab5ea8cd 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
@@ -57,12 +57,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 6,
+                        column: 5,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
index d81ee52d..92428aae 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 3,
+                        column: 2,
                     },
                 ),
                 custom: (),
@@ -53,12 +53,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 3,
+                    column: 2,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 4,
+                        column: 3,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
index 3b50a46e..efc58372 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 5,
+                    column: 4,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 10,
+                        column: 9,
                     },
                 ),
                 custom: (),
@@ -32,12 +32,12 @@ expression: parse_ast
                     left: Located {
                         location: Location {
                             row: 1,
-                            column: 2,
+                            column: 1,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 4,
+                                column: 3,
                             },
                         ),
                         custom: (),
@@ -55,12 +55,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 8,
+                                column: 7,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 10,
+                                    column: 9,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
index a8d98238..ce413379 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 5,
+                        column: 4,
                     },
                 ),
                 custom: (),
@@ -65,12 +65,12 @@ expression: parse_ast
                                     value: Located {
                                         location: Location {
                                             row: 1,
-                                            column: 2,
+                                            column: 1,
                                         },
                                         end_location: Some(
                                             Location {
                                                 row: 1,
-                                                column: 6,
+                                                column: 5,
                                             },
                                         ),
                                         custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
index 4e0f68f8..affaef80 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 4,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 8,
+                        column: 7,
                     },
                 ),
                 custom: (),
@@ -32,12 +32,12 @@ expression: parse_ast
                     left: Located {
                         location: Location {
                             row: 1,
-                            column: 2,
+                            column: 1,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 3,
+                                column: 2,
                             },
                         ),
                         custom: (),
@@ -55,12 +55,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 7,
+                                column: 6,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 8,
+                                    column: 7,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
index c4517c35..b76f7aa4 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 5,
+                        column: 4,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
index 58efec1c..c40f7e37 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
@@ -57,12 +57,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 3,
+                        column: 2,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
index d9d49f2a..d73746a8 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
@@ -57,12 +57,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 3,
+                        column: 2,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
index 55aa0d18..767ed51b 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 1,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 7,
+                        column: 6,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
index 5797f7f2..d1b75de1 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
@@ -6,7 +6,7 @@ expression: "parse_program(source, \"<test>\").unwrap()"
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
@@ -21,12 +21,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                 Located {
                     location: Location {
                         row: 1,
-                        column: 11,
+                        column: 10,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 12,
+                            column: 11,
                         },
                     ),
                     custom: (),
@@ -38,12 +38,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                 Located {
                     location: Location {
                         row: 1,
-                        column: 14,
+                        column: 13,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 15,
+                            column: 14,
                         },
                     ),
                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap
index 7d5a69fa..2c0bc513 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap
@@ -5,12 +5,12 @@ expression: parse_ast
 Located {
     location: Location {
         row: 1,
-        column: 1,
+        column: 0,
     },
     end_location: Some(
         Location {
             row: 1,
-            column: 20,
+            column: 19,
         },
     ),
     custom: (),
@@ -18,12 +18,12 @@ Located {
         key: Located {
             location: Location {
                 row: 1,
-                column: 2,
+                column: 1,
             },
             end_location: Some(
                 Location {
                     row: 1,
-                    column: 4,
+                    column: 3,
                 },
             ),
             custom: (),
@@ -35,12 +35,12 @@ Located {
         value: Located {
             location: Location {
                 row: 1,
-                column: 6,
+                column: 5,
             },
             end_location: Some(
                 Location {
                     row: 1,
-                    column: 8,
+                    column: 7,
                 },
             ),
             custom: (),
@@ -54,12 +54,12 @@ Located {
                 target: Located {
                     location: Location {
                         row: 1,
-                        column: 13,
+                        column: 12,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 14,
+                            column: 13,
                         },
                     ),
                     custom: (),
@@ -71,12 +71,12 @@ Located {
                 iter: Located {
                     location: Location {
                         row: 1,
-                        column: 18,
+                        column: 17,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 19,
+                            column: 18,
                         },
                     ),
                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
index 9cc392c8..6b04ccd9 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
@@ -5,12 +5,12 @@ expression: parse_ast
 Located {
     location: Location {
         row: 1,
-        column: 1,
+        column: 0,
     },
     end_location: Some(
         Location {
             row: 1,
-            column: 49,
+            column: 48,
         },
     ),
     custom: (),
@@ -18,12 +18,12 @@ Located {
         elt: Located {
             location: Location {
                 row: 1,
-                column: 2,
+                column: 1,
             },
             end_location: Some(
                 Location {
                     row: 1,
-                    column: 3,
+                    column: 2,
                 },
             ),
             custom: (),
@@ -37,12 +37,12 @@ Located {
                 target: Located {
                     location: Location {
                         row: 1,
-                        column: 8,
+                        column: 7,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 13,
+                            column: 12,
                         },
                     ),
                     custom: (),
@@ -51,12 +51,12 @@ Located {
                             Located {
                                 location: Location {
                                     row: 1,
-                                    column: 8,
+                                    column: 7,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 9,
+                                        column: 8,
                                     },
                                 ),
                                 custom: (),
@@ -68,12 +68,12 @@ Located {
                             Located {
                                 location: Location {
                                     row: 1,
-                                    column: 11,
+                                    column: 10,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 13,
+                                        column: 12,
                                     },
                                 ),
                                 custom: (),
@@ -89,12 +89,12 @@ Located {
                 iter: Located {
                     location: Location {
                         row: 1,
-                        column: 17,
+                        column: 16,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 18,
+                            column: 17,
                         },
                     ),
                     custom: (),
@@ -110,12 +110,12 @@ Located {
                 target: Located {
                     location: Location {
                         row: 1,
-                        column: 23,
+                        column: 22,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 24,
+                            column: 23,
                         },
                     ),
                     custom: (),
@@ -127,12 +127,12 @@ Located {
                 iter: Located {
                     location: Location {
                         row: 1,
-                        column: 28,
+                        column: 27,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 29,
+                            column: 28,
                         },
                     ),
                     custom: (),
@@ -145,12 +145,12 @@ Located {
                     Located {
                         location: Location {
                             row: 1,
-                            column: 35,
+                            column: 34,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 38,
+                                column: 37,
                             },
                         ),
                         custom: (),
@@ -158,12 +158,12 @@ Located {
                             left: Located {
                                 location: Location {
                                     row: 1,
-                                    column: 33,
+                                    column: 32,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 34,
+                                        column: 33,
                                     },
                                 ),
                                 custom: (),
@@ -179,12 +179,12 @@ Located {
                                 Located {
                                     location: Location {
                                         row: 1,
-                                        column: 37,
+                                        column: 36,
                                     },
                                     end_location: Some(
                                         Location {
                                             row: 1,
-                                            column: 38,
+                                            column: 37,
                                         },
                                     ),
                                     custom: (),
@@ -201,12 +201,12 @@ Located {
                     Located {
                         location: Location {
                             row: 1,
-                            column: 44,
+                            column: 43,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 48,
+                                column: 47,
                             },
                         ),
                         custom: (),
@@ -214,12 +214,12 @@ Located {
                             left: Located {
                                 location: Location {
                                     row: 1,
-                                    column: 42,
+                                    column: 41,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 43,
+                                        column: 42,
                                     },
                                 ),
                                 custom: (),
@@ -235,12 +235,12 @@ Located {
                                 Located {
                                     location: Location {
                                         row: 1,
-                                        column: 46,
+                                        column: 45,
                                     },
                                     end_location: Some(
                                         Location {
                                             row: 1,
-                                            column: 48,
+                                            column: 47,
                                         },
                                     ),
                                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap
index 13c2fc69..14009d32 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 15,
+                column: 14,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 15,
+                        column: 14,
                     },
                 ),
                 custom: (),
@@ -33,12 +33,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 1,
+                                column: 0,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 15,
+                                    column: 14,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap
index 1ceab196..89e05b28 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap
@@ -5,12 +5,12 @@ expression: parse_ast
 Located {
     location: Location {
         row: 1,
-        column: 1,
+        column: 0,
     },
     end_location: Some(
         Location {
             row: 1,
-            column: 15,
+            column: 14,
         },
     ),
     custom: (),
@@ -18,12 +18,12 @@ Located {
         elt: Located {
             location: Location {
                 row: 1,
-                column: 2,
+                column: 1,
             },
             end_location: Some(
                 Location {
                     row: 1,
-                    column: 3,
+                    column: 2,
                 },
             ),
             custom: (),
@@ -37,12 +37,12 @@ Located {
                 target: Located {
                     location: Location {
                         row: 1,
-                        column: 8,
+                        column: 7,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 9,
+                            column: 8,
                         },
                     ),
                     custom: (),
@@ -54,12 +54,12 @@ Located {
                 iter: Located {
                     location: Location {
                         row: 1,
-                        column: 13,
+                        column: 12,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 14,
+                            column: 13,
                         },
                     ),
                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
index c628cf98..9553a379 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
@@ -6,7 +6,7 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
@@ -19,12 +19,12 @@ expression: parse_ast
             test: Located {
                 location: Location {
                     row: 1,
-                    column: 4,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 5,
+                        column: 4,
                     },
                 ),
                 custom: (),
@@ -39,12 +39,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 7,
+                        column: 6,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 9,
+                            column: 8,
                         },
                     ),
                     custom: (),
@@ -52,12 +52,12 @@ expression: parse_ast
                         value: Located {
                             location: Location {
                                 row: 1,
-                                column: 7,
+                                column: 6,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 9,
+                                    column: 8,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
index 6f8b7c27..d9762f53 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
@@ -5,12 +5,12 @@ expression: parse_ast
 Located {
     location: Location {
         row: 1,
-        column: 1,
+        column: 0,
     },
     end_location: Some(
         Location {
             row: 1,
-            column: 27,
+            column: 26,
         },
     ),
     custom: (),
@@ -18,12 +18,12 @@ Located {
         elt: Located {
             location: Location {
                 row: 1,
-                column: 4,
+                column: 3,
             },
             end_location: Some(
                 Location {
                     row: 1,
-                    column: 15,
+                    column: 14,
                 },
             ),
             custom: (),
@@ -31,12 +31,12 @@ Located {
                 test: Located {
                     location: Location {
                         row: 1,
-                        column: 7,
+                        column: 6,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 8,
+                            column: 7,
                         },
                     ),
                     custom: (),
@@ -48,12 +48,12 @@ Located {
                 body: Located {
                     location: Location {
                         row: 1,
-                        column: 2,
+                        column: 1,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 3,
+                            column: 2,
                         },
                     ),
                     custom: (),
@@ -65,12 +65,12 @@ Located {
                 orelse: Located {
                     location: Location {
                         row: 1,
-                        column: 14,
+                        column: 13,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 15,
+                            column: 14,
                         },
                     ),
                     custom: (),
@@ -86,12 +86,12 @@ Located {
                 target: Located {
                     location: Location {
                         row: 1,
-                        column: 20,
+                        column: 19,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 21,
+                            column: 20,
                         },
                     ),
                     custom: (),
@@ -103,12 +103,12 @@ Located {
                 iter: Located {
                     location: Location {
                         row: 1,
-                        column: 25,
+                        column: 24,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 26,
+                            column: 25,
                         },
                     ),
                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap
index 61d45da8..dfe98597 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 33,
+                column: 32,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 33,
+                        column: 32,
                     },
                 ),
                 custom: (),
@@ -32,12 +32,12 @@ expression: parse_ast
                     func: Located {
                         location: Location {
                             row: 1,
-                            column: 1,
+                            column: 0,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 8,
+                                column: 7,
                             },
                         ),
                         custom: (),
@@ -50,12 +50,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 9,
+                                column: 8,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 21,
+                                    column: 20,
                                 },
                             ),
                             custom: (),
@@ -71,12 +71,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 23,
+                                column: 22,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 32,
+                                    column: 31,
                                 },
                             ),
                             custom: (),
@@ -87,12 +87,12 @@ expression: parse_ast
                                 value: Located {
                                     location: Location {
                                         row: 1,
-                                        column: 31,
+                                        column: 30,
                                     },
                                     end_location: Some(
                                         Location {
                                             row: 1,
-                                            column: 32,
+                                            column: 31,
                                         },
                                     ),
                                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap
index 7e0395a4..f305878e 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 19,
+                column: 18,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 19,
+                        column: 18,
                     },
                 ),
                 custom: (),
@@ -35,12 +35,12 @@ expression: parse_ast
                             Located {
                                 location: Location {
                                     row: 1,
-                                    column: 8,
+                                    column: 7,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 9,
+                                        column: 8,
                                     },
                                 ),
                                 custom: (),
@@ -53,12 +53,12 @@ expression: parse_ast
                             Located {
                                 location: Location {
                                     row: 1,
-                                    column: 11,
+                                    column: 10,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 12,
+                                        column: 11,
                                     },
                                 ),
                                 custom: (),
@@ -78,12 +78,12 @@ expression: parse_ast
                     body: Located {
                         location: Location {
                             row: 1,
-                            column: 16,
+                            column: 15,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 19,
+                                column: 18,
                             },
                         ),
                         custom: (),
@@ -91,12 +91,12 @@ expression: parse_ast
                             left: Located {
                                 location: Location {
                                     row: 1,
-                                    column: 14,
+                                    column: 13,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 15,
+                                        column: 14,
                                     },
                                 ),
                                 custom: (),
@@ -109,12 +109,12 @@ expression: parse_ast
                             right: Located {
                                 location: Location {
                                     row: 1,
-                                    column: 18,
+                                    column: 17,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 19,
+                                        column: 18,
                                     },
                                 ),
                                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap
index f932e7db..9af8e55e 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap
@@ -5,12 +5,12 @@ expression: parse_ast
 Located {
     location: Location {
         row: 1,
-        column: 1,
+        column: 0,
     },
     end_location: Some(
         Location {
             row: 1,
-            column: 15,
+            column: 14,
         },
     ),
     custom: (),
@@ -18,12 +18,12 @@ Located {
         elt: Located {
             location: Location {
                 row: 1,
-                column: 2,
+                column: 1,
             },
             end_location: Some(
                 Location {
                     row: 1,
-                    column: 3,
+                    column: 2,
                 },
             ),
             custom: (),
@@ -37,12 +37,12 @@ Located {
                 target: Located {
                     location: Location {
                         row: 1,
-                        column: 8,
+                        column: 7,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 9,
+                            column: 8,
                         },
                     ),
                     custom: (),
@@ -54,12 +54,12 @@ Located {
                 iter: Located {
                     location: Location {
                         row: 1,
-                        column: 13,
+                        column: 12,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 14,
+                            column: 13,
                         },
                     ),
                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
index 2f534dd8..d291cd55 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
@@ -5,12 +5,12 @@ expression: parse_ast
 Located {
     location: Location {
         row: 1,
-        column: 1,
+        column: 0,
     },
     end_location: Some(
         Location {
             row: 1,
-            column: 24,
+            column: 23,
         },
     ),
     custom: (),
@@ -18,12 +18,12 @@ Located {
         elt: Located {
             location: Location {
                 row: 1,
-                column: 2,
+                column: 1,
             },
             end_location: Some(
                 Location {
                     row: 1,
-                    column: 12,
+                    column: 11,
                 },
             ),
             custom: (),
@@ -31,12 +31,12 @@ Located {
                 target: Located {
                     location: Location {
                         row: 1,
-                        column: 2,
+                        column: 1,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 12,
+                            column: 11,
                         },
                     ),
                     custom: (),
@@ -48,12 +48,12 @@ Located {
                 value: Located {
                     location: Location {
                         row: 1,
-                        column: 9,
+                        column: 8,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 12,
+                            column: 11,
                         },
                     ),
                     custom: (),
@@ -61,12 +61,12 @@ Located {
                         left: Located {
                             location: Location {
                                 row: 1,
-                                column: 7,
+                                column: 6,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 8,
+                                    column: 7,
                                 },
                             ),
                             custom: (),
@@ -79,12 +79,12 @@ Located {
                         right: Located {
                             location: Location {
                                 row: 1,
-                                column: 11,
+                                column: 10,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 12,
+                                    column: 11,
                                 },
                             ),
                             custom: (),
@@ -104,12 +104,12 @@ Located {
                 target: Located {
                     location: Location {
                         row: 1,
-                        column: 17,
+                        column: 16,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 18,
+                            column: 17,
                         },
                     ),
                     custom: (),
@@ -121,12 +121,12 @@ Located {
                 iter: Located {
                     location: Location {
                         row: 1,
-                        column: 22,
+                        column: 21,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 23,
+                            column: 22,
                         },
                     ),
                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap
index fcb2eecc..c782b8a8 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 24,
+                column: 23,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 24,
+                        column: 23,
                     },
                 ),
                 custom: (),
@@ -32,12 +32,12 @@ expression: parse_ast
                     func: Located {
                         location: Location {
                             row: 1,
-                            column: 1,
+                            column: 0,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 6,
+                                column: 5,
                             },
                         ),
                         custom: (),
@@ -50,12 +50,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 7,
+                                column: 6,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 20,
+                                    column: 19,
                                 },
                             ),
                             custom: (),
@@ -69,12 +69,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 22,
+                                column: 21,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 23,
+                                    column: 22,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap
index 49efd477..34f6ed85 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 21,
+                column: 20,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 21,
+                        column: 20,
                     },
                 ),
                 custom: (),
@@ -32,12 +32,12 @@ expression: parse_ast
                     func: Located {
                         location: Location {
                             row: 1,
-                            column: 1,
+                            column: 0,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 6,
+                                column: 5,
                             },
                         ),
                         custom: (),
@@ -50,12 +50,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 7,
+                                column: 6,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 20,
+                                    column: 19,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap
index 318a0f9c..d58f9e3a 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 14,
+                column: 13,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 14,
+                        column: 13,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap
index c8e252c4..7be86281 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap
@@ -6,12 +6,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 12,
+                column: 11,
             },
         ),
         custom: (),
@@ -20,12 +20,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                 Located {
                     location: Location {
                         row: 1,
-                        column: 1,
+                        column: 0,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 5,
+                            column: 4,
                         },
                     ),
                     custom: (),
@@ -34,12 +34,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                             Located {
                                 location: Location {
                                     row: 1,
-                                    column: 1,
+                                    column: 0,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 2,
+                                        column: 1,
                                     },
                                 ),
                                 custom: (),
@@ -51,12 +51,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                             Located {
                                 location: Location {
                                     row: 1,
-                                    column: 4,
+                                    column: 3,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 5,
+                                        column: 4,
                                     },
                                 ),
                                 custom: (),
@@ -73,12 +73,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 8,
+                    column: 7,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 12,
+                        column: 11,
                     },
                 ),
                 custom: (),
@@ -87,12 +87,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 8,
+                                column: 7,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 9,
+                                    column: 8,
                                 },
                             ),
                             custom: (),
@@ -106,12 +106,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 11,
+                                column: 10,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 12,
+                                    column: 11,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
index 3cce5b1d..f57f903b 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 18,
+                column: 17,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 9,
+                        column: 8,
                     },
                 ),
                 custom: (),
@@ -33,12 +33,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 1,
+                                column: 0,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 9,
+                                    column: 8,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
index 3cce5b1d..f57f903b 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 18,
+                column: 17,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 9,
+                        column: 8,
                     },
                 ),
                 custom: (),
@@ -33,12 +33,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 1,
+                                column: 0,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 9,
+                                    column: 8,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
index 2ab56927..68582508 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 23,
+                column: 22,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 9,
+                        column: 8,
                     },
                 ),
                 custom: (),
@@ -33,12 +33,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 1,
+                                column: 0,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 9,
+                                    column: 8,
                                 },
                             ),
                             custom: (),
@@ -52,12 +52,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 10,
+                                column: 9,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 23,
+                                    column: 22,
                                 },
                             ),
                             custom: (),
@@ -65,12 +65,12 @@ expression: parse_ast
                                 value: Located {
                                     location: Location {
                                         row: 1,
-                                        column: 2,
+                                        column: 1,
                                     },
                                     end_location: Some(
                                         Location {
                                             row: 1,
-                                            column: 5,
+                                            column: 4,
                                         },
                                     ),
                                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
index fd88e7b3..0052f8df 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 17,
+                column: 16,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 9,
+                        column: 8,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap
index a08dde85..0616991c 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_triple_quotes_with_kind.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 21,
+                column: 20,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 21,
+                        column: 20,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
index ba2ae385..d9d57865 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 19,
+                column: 18,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 10,
+                        column: 9,
                     },
                 ),
                 custom: (),
@@ -33,12 +33,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 1,
+                                column: 0,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 10,
+                                    column: 9,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
index 7d21fea8..a9f10f06 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 23,
+                column: 22,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 10,
+                        column: 9,
                     },
                 ),
                 custom: (),
@@ -33,12 +33,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 1,
+                                column: 0,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 10,
+                                    column: 9,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
index 9c93348a..503d5b5c 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 18,
+                column: 17,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 9,
+                        column: 8,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
index 0af6e48f..43262599 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
@@ -6,12 +6,12 @@ expression: parse_ast
     Located {
         location: Location {
             row: 1,
-            column: 1,
+            column: 0,
         },
         end_location: Some(
             Location {
                 row: 1,
-                column: 18,
+                column: 17,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 0,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 10,
+                        column: 9,
                     },
                 ),
                 custom: (),

From de7acf6f99aad26cf9d7e51175c02721d0429827 Mon Sep 17 00:00:00 2001
From: dvermd <315743+dvermd@users.noreply.github.com>
Date: Mon, 17 Oct 2022 22:07:10 +0200
Subject: [PATCH 288/459] improve fstring parser

part of: #1671
---
 codegen/src/symboltable.rs |   2 +-
 parser/src/error.rs        |  34 +++++++++---
 parser/src/fstring.rs      | 104 ++++++++++++++++++++++++++-----------
 3 files changed, 102 insertions(+), 38 deletions(-)

diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 1044f981..b100a9e5 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -172,7 +172,7 @@ impl SymbolTableError {
     pub fn into_codegen_error(self, source_path: String) -> CodegenError {
         CodegenError {
             error: CodegenErrorType::SyntaxError(self.error),
-            location: self.location,
+            location: Location::new(self.location.row(), self.location.column() + 1),
             source_path,
         }
     }
diff --git a/parser/src/error.rs b/parser/src/error.rs
index 19f885d4..e8354421 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -82,25 +82,45 @@ pub enum FStringErrorType {
     InvalidExpression(Box<ParseErrorType>),
     InvalidConversionFlag,
     EmptyExpression,
-    MismatchedDelimiter,
+    MismatchedDelimiter(char, char),
     ExpressionNestedTooDeeply,
+    ExpressionCannotInclude(char),
+    SingleRbrace,
+    Unmatched(char),
+    UnterminatedString,
 }
 
 impl fmt::Display for FStringErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            FStringErrorType::UnclosedLbrace => write!(f, "Unclosed '{{'"),
+            FStringErrorType::UnclosedLbrace => write!(f, "expecting '}}'"),
             FStringErrorType::UnopenedRbrace => write!(f, "Unopened '}}'"),
             FStringErrorType::ExpectedRbrace => write!(f, "Expected '}}' after conversion flag."),
             FStringErrorType::InvalidExpression(error) => {
-                write!(f, "Invalid expression: {}", error)
+                write!(f, "{}", error)
             }
-            FStringErrorType::InvalidConversionFlag => write!(f, "Invalid conversion flag"),
-            FStringErrorType::EmptyExpression => write!(f, "Empty expression"),
-            FStringErrorType::MismatchedDelimiter => write!(f, "Mismatched delimiter"),
+            FStringErrorType::InvalidConversionFlag => write!(f, "invalid conversion character"),
+            FStringErrorType::EmptyExpression => write!(f, "empty expression not allowed"),
+            FStringErrorType::MismatchedDelimiter(first, second) => write!(
+                f,
+                "closing parenthesis '{}' does not match opening parenthesis '{}'",
+                second, first
+            ),
+            FStringErrorType::SingleRbrace => write!(f, "single '}}' is not allowed"),
+            FStringErrorType::Unmatched(delim) => write!(f, "unmatched '{}'", delim),
             FStringErrorType::ExpressionNestedTooDeeply => {
                 write!(f, "expressions nested too deeply")
             }
+            FStringErrorType::UnterminatedString => {
+                write!(f, "unterminated string")
+            }
+            FStringErrorType::ExpressionCannotInclude(c) => {
+                if *c == '\\' {
+                    write!(f, "f-string expression part cannot include a backslash")
+                } else {
+                    write!(f, "f-string expression part cannot include '{}'s", c)
+                }
+            }
         }
     }
 }
@@ -162,7 +182,7 @@ pub(crate) fn parse_error_from_lalrpop(
             let expected = (expected.len() == 1).then(|| expected[0].clone());
             ParseError {
                 error: ParseErrorType::UnrecognizedToken(token.1, expected),
-                location: token.0,
+                location: Location::new(token.0.row(), token.0.column() + 1),
                 source_path,
             }
         }
diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index 23265831..64cb7d3b 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -67,25 +67,35 @@ impl FStringParser {
                         Some('a') => ConversionFlag::Ascii,
                         Some('r') => ConversionFlag::Repr,
                         Some(_) => {
-                            return Err(InvalidConversionFlag);
+                            return Err(if expression[1..].trim().is_empty() {
+                                EmptyExpression
+                            } else {
+                                InvalidConversionFlag
+                            });
                         }
                         None => {
-                            return Err(ExpectedRbrace);
+                            return Err(if expression[1..].trim().is_empty() {
+                                EmptyExpression
+                            } else {
+                                UnclosedLbrace
+                            });
                         }
                     };
 
                     if let Some(&peek) = chars.peek() {
                         if peek != '}' && peek != ':' {
-                            if expression[1..].trim().is_empty() {
-                                return Err(EmptyExpression);
+                            return Err(if expression[1..].trim().is_empty() {
+                                EmptyExpression
                             } else {
-                                return Err(ExpectedRbrace);
-                            }
+                                UnclosedLbrace
+                            });
                         }
-                    } else if expression[1..].trim().is_empty() {
-                        return Err(EmptyExpression);
                     } else {
-                        return Err(ExpectedRbrace);
+                        return Err(if expression[1..].trim().is_empty() {
+                            EmptyExpression
+                        } else {
+                            UnclosedLbrace
+                        });
                     }
                 }
 
@@ -108,22 +118,42 @@ impl FStringParser {
                     delims.push(ch);
                 }
                 ')' => {
-                    if delims.pop() != Some('(') {
-                        return Err(MismatchedDelimiter);
+                    let last_delim = delims.pop();
+                    match last_delim {
+                        Some('(') => {
+                            expression.push(ch);
+                        }
+                        Some(c) => {
+                            return Err(MismatchedDelimiter(c, ')'));
+                        }
+                        None => {
+                            return Err(Unmatched(')'));
+                        }
                     }
-                    expression.push(ch);
                 }
                 ']' => {
-                    if delims.pop() != Some('[') {
-                        return Err(MismatchedDelimiter);
+                    let last_delim = delims.pop();
+                    match last_delim {
+                        Some('[') => {
+                            expression.push(ch);
+                        }
+                        Some(c) => {
+                            return Err(MismatchedDelimiter(c, ']'));
+                        }
+                        None => {
+                            return Err(Unmatched(']'));
+                        }
                     }
-                    expression.push(ch);
                 }
                 '}' if !delims.is_empty() => {
-                    if delims.pop() != Some('{') {
-                        return Err(MismatchedDelimiter);
+                    let last_delim = delims.pop();
+                    match last_delim {
+                        Some('{') => {
+                            expression.push(ch);
+                        }
+                        Some(c) => return Err(MismatchedDelimiter(c, '}')),
+                        None => {}
                     }
-                    expression.push(ch);
                 }
                 '}' => {
                     if expression[1..].trim().is_empty() {
@@ -171,26 +201,36 @@ impl FStringParser {
                 }
                 '"' | '\'' => {
                     expression.push(ch);
+                    let mut string_ended = false;
                     for next in &mut chars {
                         expression.push(next);
                         if next == ch {
+                            string_ended = true;
                             break;
                         }
                     }
+                    if !string_ended {
+                        return Err(UnterminatedString);
+                    }
                 }
                 ' ' if self_documenting => {
                     trailing_seq.push(ch);
                 }
+                '\\' => return Err(ExpressionCannotInclude('\\')),
                 _ => {
                     if self_documenting {
-                        return Err(ExpectedRbrace);
+                        return Err(UnclosedLbrace);
                     }
 
                     expression.push(ch);
                 }
             }
         }
-        Err(UnclosedLbrace)
+        Err(if expression[1..].trim().is_empty() {
+            EmptyExpression
+        } else {
+            UnclosedLbrace
+        })
     }
 
     fn parse_spec<'a>(
@@ -251,10 +291,14 @@ impl FStringParser {
                 '{' => {
                     chars.next();
                     if nested == 0 {
-                        if let Some('{') = chars.peek() {
-                            chars.next();
-                            content.push('{');
-                            continue;
+                        match chars.peek() {
+                            Some('{') => {
+                                chars.next();
+                                content.push('{');
+                                continue;
+                            }
+                            None => return Err(UnclosedLbrace),
+                            _ => {}
                         }
                     }
                     if !content.is_empty() {
@@ -278,7 +322,7 @@ impl FStringParser {
                         chars.next();
                         content.push('}');
                     } else {
-                        return Err(UnopenedRbrace);
+                        return Err(SingleRbrace);
                     }
                 }
                 _ => {
@@ -385,9 +429,9 @@ mod tests {
 
     #[test]
     fn test_parse_invalid_fstring() {
-        assert_eq!(parse_fstring("{5!a"), Err(ExpectedRbrace));
-        assert_eq!(parse_fstring("{5!a1}"), Err(ExpectedRbrace));
-        assert_eq!(parse_fstring("{5!"), Err(ExpectedRbrace));
+        assert_eq!(parse_fstring("{5!a"), Err(UnclosedLbrace));
+        assert_eq!(parse_fstring("{5!a1}"), Err(UnclosedLbrace));
+        assert_eq!(parse_fstring("{5!"), Err(UnclosedLbrace));
         assert_eq!(parse_fstring("abc{!a 'cat'}"), Err(EmptyExpression));
         assert_eq!(parse_fstring("{!a"), Err(EmptyExpression));
         assert_eq!(parse_fstring("{ !a}"), Err(EmptyExpression));
@@ -397,8 +441,8 @@ mod tests {
 
         assert_eq!(parse_fstring("{a:{a:{b}}}"), Err(ExpressionNestedTooDeeply));
 
-        assert_eq!(parse_fstring("{a:b}}"), Err(UnopenedRbrace));
-        assert_eq!(parse_fstring("}"), Err(UnopenedRbrace));
+        assert_eq!(parse_fstring("{a:b}}"), Err(SingleRbrace));
+        assert_eq!(parse_fstring("}"), Err(SingleRbrace));
         assert_eq!(parse_fstring("{a:{b}"), Err(UnclosedLbrace));
         assert_eq!(parse_fstring("{"), Err(UnclosedLbrace));
 

From 1dfa2e52396e211b6537a0c3e22f66e600e39092 Mon Sep 17 00:00:00 2001
From: dvermd <315743+dvermd@users.noreply.github.com>
Date: Wed, 19 Oct 2022 23:45:24 +0200
Subject: [PATCH 289/459] improve col_offset in new line and lalr

---
 core/src/location.rs                          |  2 +-
 parser/python.lalrpop                         |  2 +-
 parser/src/parser.rs                          |  2 +-
 ...on_parser__parser__tests__parse_class.snap | 34 +++++++++----------
 ...er__parser__tests__parse_if_elif_else.snap | 26 +++++++-------
 ...ed_expression_generator_comprehension.snap |  2 +-
 6 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/core/src/location.rs b/core/src/location.rs
index c4b7aa1f..7f1adc44 100644
--- a/core/src/location.rs
+++ b/core/src/location.rs
@@ -56,7 +56,7 @@ impl Location {
 
     pub fn newline(&mut self) {
         self.row += 1;
-        self.column = 1;
+        self.column = 0;
     }
 }
 
diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 310790a2..b2212a38 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -871,7 +871,7 @@ ShiftOp: ast::Operator = {
 };
 
 ArithmeticExpression: ast::Expr = {
-    <a:ArithmeticExpression> <location:@L> <op:AddOp> <b:Term> <end_location:@R> => ast::Expr {
+    <location:@L> <a:ArithmeticExpression> <op:AddOp> <b:Term> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 9bab28cf..e46aa28c 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -35,7 +35,7 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
 /// assert_eq!(
 ///     expr,
 ///     ast::Expr {
-///         location: ast::Location::new(1, 2),
+///         location: ast::Location::new(1, 0),
 ///         end_location: Some(ast::Location::new(1, 5)),
 ///         custom: (),
 ///         node: ast::ExprKind::BinOp {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
index d1b75de1..c74d2eb9 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
@@ -11,7 +11,7 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         end_location: Some(
             Location {
                 row: 5,
-                column: 7,
+                column: 6,
             },
         ),
         custom: (),
@@ -58,12 +58,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                 Located {
                     location: Location {
                         row: 2,
-                        column: 2,
+                        column: 1,
                     },
                     end_location: Some(
                         Location {
                             row: 4,
-                            column: 2,
+                            column: 1,
                         },
                     ),
                     custom: (),
@@ -75,12 +75,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                 Located {
                                     location: Location {
                                         row: 2,
-                                        column: 15,
+                                        column: 14,
                                     },
                                     end_location: Some(
                                         Location {
                                             row: 2,
-                                            column: 19,
+                                            column: 18,
                                         },
                                     ),
                                     custom: (),
@@ -101,12 +101,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                             Located {
                                 location: Location {
                                     row: 3,
-                                    column: 3,
+                                    column: 2,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 3,
-                                        column: 7,
+                                        column: 6,
                                     },
                                 ),
                                 custom: (),
@@ -121,12 +121,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                 Located {
                     location: Location {
                         row: 4,
-                        column: 2,
+                        column: 1,
                     },
                     end_location: Some(
                         Location {
                             row: 5,
-                            column: 7,
+                            column: 6,
                         },
                     ),
                     custom: (),
@@ -138,12 +138,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                 Located {
                                     location: Location {
                                         row: 4,
-                                        column: 26,
+                                        column: 25,
                                     },
                                     end_location: Some(
                                         Location {
                                             row: 4,
-                                            column: 30,
+                                            column: 29,
                                         },
                                     ),
                                     custom: (),
@@ -156,12 +156,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                 Located {
                                     location: Location {
                                         row: 4,
-                                        column: 32,
+                                        column: 31,
                                     },
                                     end_location: Some(
                                         Location {
                                             row: 4,
-                                            column: 35,
+                                            column: 34,
                                         },
                                     ),
                                     custom: (),
@@ -180,12 +180,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                 Located {
                                     location: Location {
                                         row: 4,
-                                        column: 36,
+                                        column: 35,
                                     },
                                     end_location: Some(
                                         Location {
                                             row: 4,
-                                            column: 45,
+                                            column: 44,
                                         },
                                     ),
                                     custom: (),
@@ -202,12 +202,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                             Located {
                                 location: Location {
                                     row: 5,
-                                    column: 3,
+                                    column: 2,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 5,
-                                        column: 7,
+                                        column: 6,
                                     },
                                 ),
                                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
index 9553a379..e65226d6 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
@@ -11,7 +11,7 @@ expression: parse_ast
         end_location: Some(
             Location {
                 row: 3,
-                column: 9,
+                column: 8,
             },
         ),
         custom: (),
@@ -75,12 +75,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 2,
-                        column: 1,
+                        column: 0,
                     },
                     end_location: Some(
                         Location {
                             row: 3,
-                            column: 1,
+                            column: 0,
                         },
                     ),
                     custom: (),
@@ -88,12 +88,12 @@ expression: parse_ast
                         test: Located {
                             location: Location {
                                 row: 2,
-                                column: 6,
+                                column: 5,
                             },
                             end_location: Some(
                                 Location {
                                     row: 2,
-                                    column: 7,
+                                    column: 6,
                                 },
                             ),
                             custom: (),
@@ -108,12 +108,12 @@ expression: parse_ast
                             Located {
                                 location: Location {
                                     row: 2,
-                                    column: 9,
+                                    column: 8,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 2,
-                                        column: 11,
+                                        column: 10,
                                     },
                                 ),
                                 custom: (),
@@ -121,12 +121,12 @@ expression: parse_ast
                                     value: Located {
                                         location: Location {
                                             row: 2,
-                                            column: 9,
+                                            column: 8,
                                         },
                                         end_location: Some(
                                             Location {
                                                 row: 2,
-                                                column: 11,
+                                                column: 10,
                                             },
                                         ),
                                         custom: (),
@@ -144,12 +144,12 @@ expression: parse_ast
                             Located {
                                 location: Location {
                                     row: 3,
-                                    column: 7,
+                                    column: 6,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 3,
-                                        column: 9,
+                                        column: 8,
                                     },
                                 ),
                                 custom: (),
@@ -157,12 +157,12 @@ expression: parse_ast
                                     value: Located {
                                         location: Location {
                                             row: 3,
-                                            column: 7,
+                                            column: 6,
                                         },
                                         end_location: Some(
                                             Location {
                                                 row: 3,
-                                                column: 9,
+                                                column: 8,
                                             },
                                         ),
                                         custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
index d291cd55..3bbe6c1a 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
@@ -48,7 +48,7 @@ Located {
                 value: Located {
                     location: Location {
                         row: 1,
-                        column: 8,
+                        column: 6,
                     },
                     end_location: Some(
                         Location {

From c3f246693b0b7eec70322465c76ad030c3b69513 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Sun, 6 Nov 2022 20:43:41 -0500
Subject: [PATCH 290/459] Implement Tok::Comment

---
 parser/python.lalrpop |  1 +
 parser/src/lexer.rs   | 18 ++++++++++++------
 parser/src/parser.rs  | 11 ++++++++---
 parser/src/token.rs   |  2 ++
 4 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index b2212a38..cfd6779c 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -1414,5 +1414,6 @@ extern {
         name => lexer::Tok::Name { name: <String> },
         "\n" => lexer::Tok::Newline,
         ";" => lexer::Tok::Semi,
+        "#" => lexer::Tok::Comment,
     }
 }
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 2f72cf78..37b41944 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -399,13 +399,16 @@ where
     }
 
     /// Skip everything until end of line
-    fn lex_comment(&mut self) {
+    fn lex_comment(&mut self) -> LexResult {
+        let start_pos = self.get_pos();
         self.next_char();
         loop {
             match self.chr0 {
-                Some('\n') => return,
+                Some('\n') | None => {
+                    let end_pos = self.get_pos();
+                    return Ok((start_pos, Tok::Comment, end_pos));
+                }
                 Some(_) => {}
-                None => return,
             }
             self.next_char();
         }
@@ -690,7 +693,8 @@ where
                     tabs += 1;
                 }
                 Some('#') => {
-                    self.lex_comment();
+                    let comment = self.lex_comment()?;
+                    self.emit(comment);
                     spaces = 0;
                     tabs = 0;
                 }
@@ -832,7 +836,8 @@ where
                 self.emit(number);
             }
             '#' => {
-                self.lex_comment();
+                let comment = self.lex_comment()?;
+                self.emit(comment);
             }
             '"' | '\'' => {
                 let string = self.lex_string(false, false, false, false)?;
@@ -1350,7 +1355,7 @@ mod tests {
             fn $name() {
                 let source = format!(r"99232  # {}", $eol);
                 let tokens = lex_source(&source);
-                assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Newline]);
+                assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Comment, Tok::Newline]);
             }
             )*
         }
@@ -1374,6 +1379,7 @@ mod tests {
                     tokens,
                     vec![
                         Tok::Int { value: BigInt::from(123) },
+                        Tok::Comment,
                         Tok::Newline,
                         Tok::Int { value: BigInt::from(456) },
                         Tok::Newline,
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index e46aa28c..d0f71fc8 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -5,9 +5,10 @@
 //! parse a whole program, a single statement, or a single
 //! expression.
 
-use crate::lexer::LexResult;
+use crate::lexer::{LexResult, Tok};
 pub use crate::mode::Mode;
 use crate::{ast, error::ParseError, lexer, python};
+use itertools::Itertools;
 use std::iter;
 
 /*
@@ -74,7 +75,9 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
 pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, ParseError> {
     let lxr = lexer::make_tokenizer(source);
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
-    let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
+    let tokenizer = iter::once(Ok(marker_token))
+        .chain(lxr)
+        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment));
 
     python::TopParser::new()
         .parse(tokenizer)
@@ -88,7 +91,9 @@ pub fn parse_tokens(
     source_path: &str,
 ) -> Result<ast::Mod, ParseError> {
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
-    let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
+    let tokenizer = iter::once(Ok(marker_token))
+        .chain(lxr)
+        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment));
 
     python::TopParser::new()
         .parse(tokenizer)
diff --git a/parser/src/token.rs b/parser/src/token.rs
index 5fd6728b..ba76f6a4 100644
--- a/parser/src/token.rs
+++ b/parser/src/token.rs
@@ -25,6 +25,7 @@ pub enum Tok {
     Rsqb,
     Colon,
     Comma,
+    Comment,
     Semi,
     Plus,
     Minus,
@@ -155,6 +156,7 @@ impl fmt::Display for Tok {
             Rsqb => f.write_str("']'"),
             Colon => f.write_str("':'"),
             Comma => f.write_str("','"),
+            Comment => f.write_str("#"),
             Semi => f.write_str("';'"),
             Plus => f.write_str("'+'"),
             Minus => f.write_str("'-'"),

From ac1821b23d3167ea24339e21f43f248b63e766d2 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Fri, 4 Nov 2022 16:42:17 -0400
Subject: [PATCH 291/459] Set comparator start location to beginning of
 comparison

---
 parser/python.lalrpop                                         | 2 +-
 ...stpython_parser__fstring__tests__parse_fstring_equals.snap | 2 +-
 ...thon_parser__fstring__tests__parse_fstring_not_equals.snap | 2 +-
 ...arser__parser__tests__parse_double_list_comprehension.snap | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index b2212a38..96639c11 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -800,7 +800,7 @@ NotTest: ast::Expr = {
 };
 
 Comparison: ast::Expr = {
-    <left:Expression> <location:@L> <comparisons:(CompOp Expression)+> <end_location:@R> => {
+    <location:@L> <left:Expression> <comparisons:(CompOp Expression)+> <end_location:@R> => {
         let (ops, comparators) = comparisons.into_iter().unzip();
         ast::Expr {
             location,
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
index efc58372..fbbe8492 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
@@ -19,7 +19,7 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 4,
+                    column: 1,
                 },
                 end_location: Some(
                     Location {
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
index affaef80..8f667497 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
@@ -19,7 +19,7 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 3,
+                    column: 1,
                 },
                 end_location: Some(
                     Location {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
index 6b04ccd9..f8346b5c 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap
@@ -145,7 +145,7 @@ Located {
                     Located {
                         location: Location {
                             row: 1,
-                            column: 34,
+                            column: 32,
                         },
                         end_location: Some(
                             Location {
@@ -201,7 +201,7 @@ Located {
                     Located {
                         location: Location {
                             row: 1,
-                            column: 43,
+                            column: 41,
                         },
                         end_location: Some(
                             Location {

From e081f1041e9c14e95237751c116a19d05dda722b Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Thu, 10 Nov 2022 15:39:31 +0900
Subject: [PATCH 292/459] Apply let-else statements

---
 codegen/src/compile.rs | 4 +---
 parser/build.rs        | 8 ++------
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 6666d231..7bf61692 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -1412,9 +1412,7 @@ impl Compiler {
     ) -> CompileResult<()> {
         let with_location = self.current_source_location;
 
-        let (item, items) = if let Some(parts) = items.split_first() {
-            parts
-        } else {
+        let Some((item, items)) = items.split_first() else {
             return Err(self.error(CodegenErrorType::EmptyWithItems));
         };
 
diff --git a/parser/build.rs b/parser/build.rs
index 81c28f43..4559f263 100644
--- a/parser/build.rs
+++ b/parser/build.rs
@@ -17,9 +17,7 @@ fn main() -> anyhow::Result<()> {
 }
 
 fn requires_lalrpop(source: &str, target: &str) -> Option<String> {
-    let target = if let Ok(target) = File::open(target) {
-        target
-    } else {
+    let Ok(target) = File::open(target) else {
         return Some("python.rs doesn't exist. regenerate.".to_owned());
     };
 
@@ -71,9 +69,7 @@ fn requires_lalrpop(source: &str, target: &str) -> Option<String> {
 }
 
 fn try_lalrpop(source: &str, target: &str) -> anyhow::Result<()> {
-    let _message = if let Some(msg) = requires_lalrpop(source, target) {
-        msg
-    } else {
+    let Some(_message) = requires_lalrpop(source, target) else {
         return Ok(());
     };
 

From 7cdcc13341753f3ae11a55d9bcb669c81874ef85 Mon Sep 17 00:00:00 2001
From: Bongjun Jang <bongjun.jang@kaist.ac.kr>
Date: Sat, 19 Nov 2022 21:43:34 +0900
Subject: [PATCH 293/459] Refactor lexer struct (#4257)

---
 parser/src/lexer.rs | 363 ++++++++++++++++++++++++++------------------
 1 file changed, 219 insertions(+), 144 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 37b41944..7bf88851 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -11,6 +11,8 @@ use num_traits::identities::Zero;
 use num_traits::Num;
 use std::char;
 use std::cmp::Ordering;
+use std::ops::Index;
+use std::slice::SliceIndex;
 use std::str::FromStr;
 use unic_emoji_char::is_emoji_presentation;
 use unic_ucd_ident::{is_xid_continue, is_xid_start};
@@ -56,15 +58,90 @@ impl IndentationLevel {
     }
 }
 
+#[derive(Debug)]
+struct Indentations {
+    indent_stack: Vec<IndentationLevel>,
+}
+
+impl Indentations {
+    fn is_empty(&self) -> bool {
+        self.indent_stack.len() == 1
+    }
+
+    fn push(&mut self, indent: IndentationLevel) {
+        self.indent_stack.push(indent);
+    }
+
+    fn pop(&mut self) -> Option<IndentationLevel> {
+        if self.is_empty() {
+            return None;
+        }
+        self.indent_stack.pop()
+    }
+
+    fn current(&self) -> &IndentationLevel {
+        self.indent_stack
+            .last()
+            .expect("Indetations must have at least one level")
+    }
+}
+
+impl Default for Indentations {
+    fn default() -> Self {
+        Self {
+            indent_stack: vec![IndentationLevel::default()],
+        }
+    }
+}
+
+struct CharWindow<T: Iterator<Item = char>, const N: usize> {
+    source: T,
+    window: [Option<char>; N],
+}
+
+impl<T, const N: usize> CharWindow<T, N>
+where
+    T: Iterator<Item = char>,
+{
+    fn new(source: T) -> Self {
+        Self {
+            source,
+            window: [None; N],
+        }
+    }
+
+    fn slide(&mut self) -> Option<char> {
+        self.window.rotate_left(1);
+        let next = self.source.next();
+        *self.window.last_mut().expect("never empty") = next;
+        next
+    }
+
+    fn change_first(&mut self, ch: char) {
+        *self.window.first_mut().expect("never empty") = Some(ch);
+    }
+}
+
+impl<T, const N: usize, Idx> Index<Idx> for CharWindow<T, N>
+where
+    T: Iterator<Item = char>,
+    Idx: SliceIndex<[Option<char>], Output = Option<char>>,
+{
+    type Output = Option<char>;
+
+    fn index(&self, index: Idx) -> &Self::Output {
+        self.window.index(index)
+    }
+}
+
 pub struct Lexer<T: Iterator<Item = char>> {
-    chars: T,
+    window: CharWindow<T, 3>,
+
     at_begin_of_line: bool,
     nesting: usize, // Amount of parenthesis
-    indentation_stack: Vec<IndentationLevel>,
+    indentations: Indentations,
+
     pending: Vec<Spanned>,
-    chr0: Option<char>,
-    chr1: Option<char>,
-    chr2: Option<char>,
     location: Location,
 }
 
@@ -91,9 +168,7 @@ pub fn make_tokenizer_located(
 // The newline handler is an iterator which collapses different newline
 // types into \n always.
 pub struct NewlineHandler<T: Iterator<Item = char>> {
-    source: T,
-    chr0: Option<char>,
-    chr1: Option<char>,
+    window: CharWindow<T, 2>,
 }
 
 impl<T> NewlineHandler<T>
@@ -102,9 +177,7 @@ where
 {
     pub fn new(source: T) -> Self {
         let mut nlh = NewlineHandler {
-            source,
-            chr0: None,
-            chr1: None,
+            window: CharWindow::new(source),
         };
         nlh.shift();
         nlh.shift();
@@ -112,9 +185,8 @@ where
     }
 
     fn shift(&mut self) -> Option<char> {
-        let result = self.chr0;
-        self.chr0 = self.chr1;
-        self.chr1 = self.source.next();
+        let result = self.window[0];
+        self.window.slide();
         result
     }
 }
@@ -128,16 +200,16 @@ where
     fn next(&mut self) -> Option<Self::Item> {
         // Collapse \r\n into \n
         loop {
-            if self.chr0 == Some('\r') {
-                if self.chr1 == Some('\n') {
-                    // Transform windows EOL into \n
+            match (self.window[0], self.window[1]) {
+                (Some('\r'), Some('\n')) => {
+                    // Windows EOL into \n
                     self.shift();
-                } else {
-                    // Transform MAC EOL into \n
-                    self.chr0 = Some('\n')
                 }
-            } else {
-                break;
+                (Some('\r'), _) => {
+                    // MAC EOL into \n
+                    self.window.change_first('\n');
+                }
+                _ => break,
             }
         }
 
@@ -154,19 +226,16 @@ where
 {
     pub fn new(input: T, start: Location) -> Self {
         let mut lxr = Lexer {
-            chars: input,
             at_begin_of_line: true,
             nesting: 0,
-            indentation_stack: vec![Default::default()],
+            indentations: Indentations::default(),
             pending: Vec::new(),
-            chr0: None,
             location: start,
-            chr1: None,
-            chr2: None,
+            window: CharWindow::new(input),
         };
-        lxr.next_char();
-        lxr.next_char();
-        lxr.next_char();
+        lxr.window.slide();
+        lxr.window.slide();
+        lxr.window.slide();
         // Start at top row (=1) left column (=1)
         lxr.location.reset();
         lxr
@@ -184,17 +253,15 @@ where
         let mut saw_f = false;
         loop {
             // Detect r"", f"", b"" and u""
-            if !(saw_b || saw_u || saw_f) && matches!(self.chr0, Some('b') | Some('B')) {
+            if !(saw_b || saw_u || saw_f) && matches!(self.window[0], Some('b' | 'B')) {
                 saw_b = true;
             } else if !(saw_b || saw_r || saw_u || saw_f)
-                && matches!(self.chr0, Some('u') | Some('U'))
+                && matches!(self.window[0], Some('u' | 'U'))
             {
                 saw_u = true;
-            } else if !(saw_r || saw_u) && (self.chr0 == Some('r') || self.chr0 == Some('R')) {
+            } else if !(saw_r || saw_u) && matches!(self.window[0], Some('r' | 'R')) {
                 saw_r = true;
-            } else if !(saw_b || saw_u || saw_f)
-                && (self.chr0 == Some('f') || self.chr0 == Some('F'))
-            {
+            } else if !(saw_b || saw_u || saw_f) && matches!(self.window[0], Some('f' | 'F')) {
                 saw_f = true;
             } else {
                 break;
@@ -204,7 +271,7 @@ where
             name.push(self.next_char().unwrap());
 
             // Check if we have a string:
-            if self.chr0 == Some('"') || self.chr0 == Some('\'') {
+            if matches!(self.window[0], Some('"' | '\'')) {
                 return self
                     .lex_string(saw_b, saw_r, saw_u, saw_f)
                     .map(|(_, tok, end_pos)| (start_pos, tok, end_pos));
@@ -226,19 +293,19 @@ where
     /// Numeric lexing. The feast can start!
     fn lex_number(&mut self) -> LexResult {
         let start_pos = self.get_pos();
-        if self.chr0 == Some('0') {
-            if self.chr1 == Some('x') || self.chr1 == Some('X') {
-                // Hex!
+        if self.window[0] == Some('0') {
+            if matches!(self.window[1], Some('x' | 'X')) {
+                // Hex! (0xdeadbeef)
                 self.next_char();
                 self.next_char();
                 self.lex_number_radix(start_pos, 16)
-            } else if self.chr1 == Some('o') || self.chr1 == Some('O') {
-                // Octal style!
+            } else if matches!(self.window[1], Some('o' | 'O')) {
+                // Octal style! (0o377)
                 self.next_char();
                 self.next_char();
                 self.lex_number_radix(start_pos, 8)
-            } else if self.chr1 == Some('b') || self.chr1 == Some('B') {
-                // Binary!
+            } else if matches!(self.window[1], Some('b' | 'B')) {
+                // Binary! (0b_1110_0101)
                 self.next_char();
                 self.next_char();
                 self.lex_number_radix(start_pos, 2)
@@ -264,15 +331,15 @@ where
     /// Lex a normal number, that is, no octal, hex or binary number.
     fn lex_normal_number(&mut self) -> LexResult {
         let start_pos = self.get_pos();
-        let start_is_zero = self.chr0 == Some('0');
+        let start_is_zero = self.window[0] == Some('0');
         // Normal number:
         let mut value_text = self.radix_run(10);
 
         // If float:
-        if self.chr0 == Some('.') || self.at_exponent() {
+        if self.window[0] == Some('.') || self.at_exponent() {
             // Take '.':
-            if self.chr0 == Some('.') {
-                if self.chr1 == Some('_') {
+            if self.window[0] == Some('.') {
+                if self.window[1] == Some('_') {
                     return Err(LexicalError {
                         error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),
                         location: self.get_pos(),
@@ -283,8 +350,8 @@ where
             }
 
             // 1e6 for example:
-            if self.chr0 == Some('e') || self.chr0 == Some('E') {
-                if self.chr1 == Some('_') {
+            if let Some('e' | 'E') = self.window[0] {
+                if self.window[1] == Some('_') {
                     return Err(LexicalError {
                         error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),
                         location: self.get_pos(),
@@ -292,8 +359,8 @@ where
                 }
                 value_text.push(self.next_char().unwrap().to_ascii_lowercase());
                 // Optional +/-
-                if self.chr0 == Some('-') || self.chr0 == Some('+') {
-                    if self.chr1 == Some('_') {
+                if matches!(self.window[0], Some('-' | '+')) {
+                    if self.window[1] == Some('_') {
                         return Err(LexicalError {
                             error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),
                             location: self.get_pos(),
@@ -309,8 +376,9 @@ where
                 error: LexicalErrorType::OtherError("Invalid decimal literal".to_owned()),
                 location: self.get_pos(),
             })?;
+
             // Parse trailing 'j':
-            if self.chr0 == Some('j') || self.chr0 == Some('J') {
+            if matches!(self.window[0], Some('j' | 'J')) {
                 self.next_char();
                 let end_pos = self.get_pos();
                 Ok((
@@ -327,7 +395,7 @@ where
             }
         } else {
             // Parse trailing 'j':
-            if self.chr0 == Some('j') || self.chr0 == Some('J') {
+            if matches!(self.window[0], Some('j' | 'J')) {
                 self.next_char();
                 let end_pos = self.get_pos();
                 let imag = f64::from_str(&value_text).unwrap();
@@ -336,6 +404,7 @@ where
                 let end_pos = self.get_pos();
                 let value = value_text.parse::<BigInt>().unwrap();
                 if start_is_zero && !value.is_zero() {
+                    // leading zeros in decimal integer literals are not permitted
                     return Err(LexicalError {
                         error: LexicalErrorType::OtherError("Invalid Token".to_owned()),
                         location: self.get_pos(),
@@ -355,7 +424,9 @@ where
         loop {
             if let Some(c) = self.take_number(radix) {
                 value_text.push(c);
-            } else if self.chr0 == Some('_') && Lexer::<T>::is_digit_of_radix(self.chr1, radix) {
+            } else if self.window[0] == Some('_')
+                && Lexer::<T>::is_digit_of_radix(self.window[1], radix)
+            {
                 self.next_char();
             } else {
                 break;
@@ -366,7 +437,7 @@ where
 
     /// Consume a single character with the given radix.
     fn take_number(&mut self, radix: u32) -> Option<char> {
-        let take_char = Lexer::<T>::is_digit_of_radix(self.chr0, radix);
+        let take_char = Lexer::<T>::is_digit_of_radix(self.window[0], radix);
 
         if take_char {
             Some(self.next_char().unwrap())
@@ -388,9 +459,9 @@ where
 
     /// Test if we face '[eE][-+]?[0-9]+'
     fn at_exponent(&self) -> bool {
-        match self.chr0 {
-            Some('e') | Some('E') => match self.chr1 {
-                Some('+') | Some('-') => matches!(self.chr2, Some('0'..='9')),
+        match self.window[0] {
+            Some('e' | 'E') => match self.window[1] {
+                Some('+' | '-') => matches!(self.window[2], Some('0'..='9')),
                 Some('0'..='9') => true,
                 _ => false,
             },
@@ -403,7 +474,7 @@ where
         let start_pos = self.get_pos();
         self.next_char();
         loop {
-            match self.chr0 {
+            match self.window[0] {
                 Some('\n') | None => {
                     let end_pos = self.get_pos();
                     return Ok((start_pos, Tok::Comment, end_pos));
@@ -439,7 +510,7 @@ where
         let mut octet_content = String::new();
         octet_content.push(first);
         while octet_content.len() < 3 {
-            if let Some('0'..='7') = self.chr0 {
+            if let Some('0'..='7') = self.window[0] {
                 octet_content.push(self.next_char().unwrap())
             } else {
                 break;
@@ -501,18 +572,19 @@ where
 
         // If the next two characters are also the quote character, then we have a triple-quoted
         // string; consume those two characters and ensure that we require a triple-quote to close
-        let triple_quoted = if self.chr0 == Some(quote_char) && self.chr1 == Some(quote_char) {
-            self.next_char();
-            self.next_char();
-            true
-        } else {
-            false
-        };
+        let triple_quoted =
+            if self.window[0] == Some(quote_char) && self.window[1] == Some(quote_char) {
+                self.next_char();
+                self.next_char();
+                true
+            } else {
+                false
+            };
 
         loop {
             match self.next_char() {
                 Some('\\') => {
-                    if self.chr0 == Some(quote_char) && !is_raw {
+                    if self.window[0] == Some(quote_char) && !is_raw {
                         string_content.push(quote_char);
                         self.next_char();
                     } else if is_raw {
@@ -572,7 +644,9 @@ where
                             // Look ahead at the next two characters; if we have two more
                             // quote_chars, it's the end of the string; consume the remaining
                             // closing quotes and break the loop
-                            if self.chr0 == Some(quote_char) && self.chr1 == Some(quote_char) {
+                            if self.window[0] == Some(quote_char)
+                                && self.window[1] == Some(quote_char)
+                            {
                                 self.next_char();
                                 self.next_char();
                                 break;
@@ -631,7 +705,7 @@ where
     }
 
     fn is_identifier_continuation(&self) -> bool {
-        if let Some(c) = self.chr0 {
+        if let Some(c) = self.window[0] {
             match c {
                 '_' | '0'..='9' => true,
                 c => is_xid_continue(c),
@@ -663,7 +737,7 @@ where
         let mut spaces: usize = 0;
         let mut tabs: usize = 0;
         loop {
-            match self.chr0 {
+            match self.window[0] {
                 Some(' ') => {
                     /*
                     if tabs != 0 {
@@ -729,44 +803,46 @@ where
     fn handle_indentations(&mut self) -> Result<(), LexicalError> {
         let indentation_level = self.eat_indentation()?;
 
-        if self.nesting == 0 {
-            // Determine indent or dedent:
-            let current_indentation = self.indentation_stack.last().unwrap();
-            let ordering = indentation_level.compare_strict(current_indentation, self.get_pos())?;
-            match ordering {
-                Ordering::Equal => {
-                    // Same same
-                }
-                Ordering::Greater => {
-                    // New indentation level:
-                    self.indentation_stack.push(indentation_level);
-                    let tok_pos = self.get_pos();
-                    self.emit((tok_pos, Tok::Indent, tok_pos));
-                }
-                Ordering::Less => {
-                    // One or more dedentations
-                    // Pop off other levels until col is found:
-
-                    loop {
-                        let current_indentation = self.indentation_stack.last().unwrap();
-                        let ordering = indentation_level
-                            .compare_strict(current_indentation, self.get_pos())?;
-                        match ordering {
-                            Ordering::Less => {
-                                self.indentation_stack.pop();
-                                let tok_pos = self.get_pos();
-                                self.emit((tok_pos, Tok::Dedent, tok_pos));
-                            }
-                            Ordering::Equal => {
-                                // We arrived at proper level of indentation.
-                                break;
-                            }
-                            Ordering::Greater => {
-                                return Err(LexicalError {
-                                    error: LexicalErrorType::IndentationError,
-                                    location: self.get_pos(),
-                                });
-                            }
+        if self.nesting != 0 {
+            return Ok(());
+        }
+
+        // Determine indent or dedent:
+        let current_indentation = self.indentations.current();
+        let ordering = indentation_level.compare_strict(current_indentation, self.get_pos())?;
+        match ordering {
+            Ordering::Equal => {
+                // Same same
+            }
+            Ordering::Greater => {
+                // New indentation level:
+                self.indentations.push(indentation_level);
+                let tok_pos = self.get_pos();
+                self.emit((tok_pos, Tok::Indent, tok_pos));
+            }
+            Ordering::Less => {
+                // One or more dedentations
+                // Pop off other levels until col is found:
+
+                loop {
+                    let current_indentation = self.indentations.current();
+                    let ordering =
+                        indentation_level.compare_strict(current_indentation, self.get_pos())?;
+                    match ordering {
+                        Ordering::Less => {
+                            self.indentations.pop();
+                            let tok_pos = self.get_pos();
+                            self.emit((tok_pos, Tok::Dedent, tok_pos));
+                        }
+                        Ordering::Equal => {
+                            // We arrived at proper level of indentation.
+                            break;
+                        }
+                        Ordering::Greater => {
+                            return Err(LexicalError {
+                                error: LexicalErrorType::IndentationError,
+                                location: self.get_pos(),
+                            });
                         }
                     }
                 }
@@ -779,7 +855,7 @@ where
     /// Take a look at the next character, if any, and decide upon the next steps.
     fn consume_normal(&mut self) -> Result<(), LexicalError> {
         // Check if we have some character:
-        if let Some(c) = self.chr0 {
+        if let Some(c) = self.window[0] {
             // First check identifier:
             if self.is_identifier_start(c) {
                 let identifier = self.lex_identifier()?;
@@ -817,8 +893,8 @@ where
             }
 
             // Next, flush the indentation stack to zero.
-            while self.indentation_stack.len() > 1 {
-                self.indentation_stack.pop();
+            while !self.indentations.is_empty() {
+                self.indentations.pop();
                 self.emit((tok_pos, Tok::Dedent, tok_pos));
             }
 
@@ -846,7 +922,7 @@ where
             '=' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                match self.chr0 {
+                match self.window[0] {
                     Some('=') => {
                         self.next_char();
                         let tok_end = self.get_pos();
@@ -861,7 +937,7 @@ where
             '+' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                if let Some('=') = self.chr0 {
+                if let Some('=') = self.window[0] {
                     self.next_char();
                     let tok_end = self.get_pos();
                     self.emit((tok_start, Tok::PlusEqual, tok_end));
@@ -873,7 +949,7 @@ where
             '*' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                match self.chr0 {
+                match self.window[0] {
                     Some('=') => {
                         self.next_char();
                         let tok_end = self.get_pos();
@@ -881,7 +957,7 @@ where
                     }
                     Some('*') => {
                         self.next_char();
-                        match self.chr0 {
+                        match self.window[0] {
                             Some('=') => {
                                 self.next_char();
                                 let tok_end = self.get_pos();
@@ -902,7 +978,7 @@ where
             '/' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                match self.chr0 {
+                match self.window[0] {
                     Some('=') => {
                         self.next_char();
                         let tok_end = self.get_pos();
@@ -910,7 +986,7 @@ where
                     }
                     Some('/') => {
                         self.next_char();
-                        match self.chr0 {
+                        match self.window[0] {
                             Some('=') => {
                                 self.next_char();
                                 let tok_end = self.get_pos();
@@ -931,7 +1007,7 @@ where
             '%' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                if let Some('=') = self.chr0 {
+                if let Some('=') = self.window[0] {
                     self.next_char();
                     let tok_end = self.get_pos();
                     self.emit((tok_start, Tok::PercentEqual, tok_end));
@@ -943,7 +1019,7 @@ where
             '|' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                if let Some('=') = self.chr0 {
+                if let Some('=') = self.window[0] {
                     self.next_char();
                     let tok_end = self.get_pos();
                     self.emit((tok_start, Tok::VbarEqual, tok_end));
@@ -955,7 +1031,7 @@ where
             '^' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                if let Some('=') = self.chr0 {
+                if let Some('=') = self.window[0] {
                     self.next_char();
                     let tok_end = self.get_pos();
                     self.emit((tok_start, Tok::CircumflexEqual, tok_end));
@@ -967,7 +1043,7 @@ where
             '&' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                if let Some('=') = self.chr0 {
+                if let Some('=') = self.window[0] {
                     self.next_char();
                     let tok_end = self.get_pos();
                     self.emit((tok_start, Tok::AmperEqual, tok_end));
@@ -979,7 +1055,7 @@ where
             '-' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                match self.chr0 {
+                match self.window[0] {
                     Some('=') => {
                         self.next_char();
                         let tok_end = self.get_pos();
@@ -999,7 +1075,7 @@ where
             '@' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                if let Some('=') = self.chr0 {
+                if let Some('=') = self.window[0] {
                     self.next_char();
                     let tok_end = self.get_pos();
                     self.emit((tok_start, Tok::AtEqual, tok_end));
@@ -1011,7 +1087,7 @@ where
             '!' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                if let Some('=') = self.chr0 {
+                if let Some('=') = self.window[0] {
                     self.next_char();
                     let tok_end = self.get_pos();
                     self.emit((tok_start, Tok::NotEqual, tok_end));
@@ -1070,7 +1146,7 @@ where
             ':' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                if let Some('=') = self.chr0 {
+                if let Some('=') = self.window[0] {
                     self.next_char();
                     let tok_end = self.get_pos();
                     self.emit((tok_start, Tok::ColonEqual, tok_end));
@@ -1085,10 +1161,10 @@ where
             '<' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                match self.chr0 {
+                match self.window[0] {
                     Some('<') => {
                         self.next_char();
-                        match self.chr0 {
+                        match self.window[0] {
                             Some('=') => {
                                 self.next_char();
                                 let tok_end = self.get_pos();
@@ -1114,10 +1190,10 @@ where
             '>' => {
                 let tok_start = self.get_pos();
                 self.next_char();
-                match self.chr0 {
+                match self.window[0] {
                     Some('>') => {
                         self.next_char();
-                        match self.chr0 {
+                        match self.window[0] {
                             Some('=') => {
                                 self.next_char();
                                 let tok_end = self.get_pos();
@@ -1147,13 +1223,13 @@ where
                 self.emit((tok_start, Tok::Comma, tok_end));
             }
             '.' => {
-                if let Some('0'..='9') = self.chr1 {
+                if let Some('0'..='9') = self.window[1] {
                     let number = self.lex_number()?;
                     self.emit(number);
                 } else {
                     let tok_start = self.get_pos();
                     self.next_char();
-                    if let (Some('.'), Some('.')) = (&self.chr0, &self.chr1) {
+                    if let (Some('.'), Some('.')) = (&self.window[0], &self.window[1]) {
                         self.next_char();
                         self.next_char();
                         let tok_end = self.get_pos();
@@ -1178,14 +1254,13 @@ where
             ' ' | '\t' | '\x0C' => {
                 // Skip whitespaces
                 self.next_char();
-                while self.chr0 == Some(' ') || self.chr0 == Some('\t') || self.chr0 == Some('\x0C')
-                {
+                while let Some(' ' | '\t' | '\x0C') = self.window[0] {
                     self.next_char();
                 }
             }
             '\\' => {
                 self.next_char();
-                if let Some('\n') = self.chr0 {
+                if let Some('\n') = self.window[0] {
                     self.next_char();
                 } else {
                     return Err(LexicalError {
@@ -1194,7 +1269,7 @@ where
                     });
                 }
 
-                if self.chr0.is_none() {
+                if self.window[0].is_none() {
                     return Err(LexicalError {
                         error: LexicalErrorType::Eof,
                         location: self.get_pos(),
@@ -1223,11 +1298,8 @@ where
 
     /// Helper function to go to the next character coming up.
     fn next_char(&mut self) -> Option<char> {
-        let c = self.chr0;
-        let nxt = self.chars.next();
-        self.chr0 = self.chr1;
-        self.chr1 = self.chr2;
-        self.chr2 = nxt;
+        let c = self.window[0];
+        self.window.slide();
         if c == Some('\n') {
             self.location.newline();
         } else {
@@ -1267,7 +1339,7 @@ where
             "Lex token {:?}, nesting={:?}, indent stack: {:?}",
             token,
             self.nesting,
-            self.indentation_stack
+            self.indentations,
         );
 
         match token {
@@ -1317,7 +1389,7 @@ mod tests {
 
     #[test]
     fn test_numbers() {
-        let source = "0x2f 0b1101 0 123 0.2 2j 2.2j";
+        let source = "0x2f 0b1101 0 123 123_45_67_890 0.2 2j 2.2j";
         let tokens = lex_source(source);
         assert_eq!(
             tokens,
@@ -1334,6 +1406,9 @@ mod tests {
                 Tok::Int {
                     value: BigInt::from(123),
                 },
+                Tok::Int {
+                    value: BigInt::from(1234567890),
+                },
                 Tok::Float { value: 0.2 },
                 Tok::Complex {
                     real: 0.0,

From 7f44441183b49cfb2c9a5f3cfb5c8a7f32558da4 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Sat, 19 Nov 2022 09:39:22 -0500
Subject: [PATCH 294/459] Implement some minor performance optimizations

---
 parser/Cargo.toml      |  1 +
 parser/src/function.rs |  6 +++---
 parser/src/string.rs   | 13 +++++++++++++
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index b27bb500..26241c40 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -32,6 +32,7 @@ unic-emoji-char = "0.9.0"
 unic-ucd-ident  = "0.9.0"
 unicode_names2 = "0.5.0"
 thiserror = "1.0"
+fnv = "1.0.7"
 
 [dev-dependencies]
 insta = "1.14.0"
diff --git a/parser/src/function.rs b/parser/src/function.rs
index 28cb44fe..b2b24a88 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -1,7 +1,6 @@
 use crate::ast;
 use crate::error::{LexicalError, LexicalErrorType};
-use ahash::RandomState;
-use std::collections::HashSet;
+use fnv::FnvHashSet;
 
 pub struct ArgumentList {
     pub args: Vec<ast::Expr>,
@@ -54,7 +53,8 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, Lexi
     let mut args = vec![];
     let mut keywords = vec![];
 
-    let mut keyword_names = HashSet::with_capacity_and_hasher(func_args.len(), RandomState::new());
+    let mut keyword_names =
+        FnvHashSet::with_capacity_and_hasher(func_args.len(), Default::default());
     for (name, value) in func_args {
         match name {
             Some((start, end, name)) => {
diff --git a/parser/src/string.rs b/parser/src/string.rs
index 078ac1e2..e76821d5 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -14,6 +14,19 @@ pub fn parse_strings(
     let initial_end = values[0].2;
     let initial_kind = (values[0].1 .1 == StringKind::U).then(|| "u".to_owned());
 
+    // Optimization: fast-track the common case of a single string.
+    if values.len() == 1 && matches!(&values[0].1 .1, StringKind::Normal | StringKind::U) {
+        let value = values.into_iter().last().unwrap().1 .0;
+        return Ok(Expr::new(
+            initial_start,
+            initial_end,
+            ExprKind::Constant {
+                value: Constant::Str(value),
+                kind: initial_kind,
+            },
+        ));
+    }
+
     // Determine whether the list of values contains any f-strings. (If not, we can return a
     // single Constant at the end, rather than a JoinedStr.)
     let mut has_fstring = false;

From d9b7daa844fb3b93af59eaa3f9d6534b4f8986e3 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Sun, 20 Nov 2022 15:30:19 -0500
Subject: [PATCH 295/459] Use rustc-hash

---
 parser/Cargo.toml      | 4 ++--
 parser/src/function.rs | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index 26241c40..e3ce5a39 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -28,11 +28,11 @@ log = "0.4.16"
 num-bigint = "0.4.3"
 num-traits = "0.2.14"
 phf = "0.10.1"
+rustc-hash = "1.1.0"
+thiserror = "1.0"
 unic-emoji-char = "0.9.0"
 unic-ucd-ident  = "0.9.0"
 unicode_names2 = "0.5.0"
-thiserror = "1.0"
-fnv = "1.0.7"
 
 [dev-dependencies]
 insta = "1.14.0"
diff --git a/parser/src/function.rs b/parser/src/function.rs
index b2b24a88..bc4d5142 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -1,6 +1,6 @@
 use crate::ast;
 use crate::error::{LexicalError, LexicalErrorType};
-use fnv::FnvHashSet;
+use rustc_hash::FxHashSet;
 
 pub struct ArgumentList {
     pub args: Vec<ast::Expr>,
@@ -54,7 +54,7 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, Lexi
     let mut keywords = vec![];
 
     let mut keyword_names =
-        FnvHashSet::with_capacity_and_hasher(func_args.len(), Default::default());
+        FxHashSet::with_capacity_and_hasher(func_args.len(), Default::default());
     for (name, value) in func_args {
         match name {
             Some((start, end, name)) => {

From 2fb2224632a37bd17af2b01cc7198d11d0adfa9a Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Sun, 20 Nov 2022 23:41:52 -0500
Subject: [PATCH 296/459] Use match

---
 parser/src/string.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/src/string.rs b/parser/src/string.rs
index e76821d5..838da94c 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -15,7 +15,7 @@ pub fn parse_strings(
     let initial_kind = (values[0].1 .1 == StringKind::U).then(|| "u".to_owned());
 
     // Optimization: fast-track the common case of a single string.
-    if values.len() == 1 && matches!(&values[0].1 .1, StringKind::Normal | StringKind::U) {
+    if matches!(&*values, [(_, (_, StringKind::Normal | StringKind::U), _)]) {
         let value = values.into_iter().last().unwrap().1 .0;
         return Ok(Expr::new(
             initial_start,

From 1fe79331f3788199ab13cd87a2f89d5ef5ca486d Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Mon, 21 Nov 2022 22:36:05 +0900
Subject: [PATCH 297/459] Add test

---
 parser/src/lexer.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 7bf88851..62bc4ca3 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -1389,7 +1389,7 @@ mod tests {
 
     #[test]
     fn test_numbers() {
-        let source = "0x2f 0b1101 0 123 123_45_67_890 0.2 2j 2.2j";
+        let source = "0x2f 0o12 0b1101 0 123 123_45_67_890 0.2 1e+2 2.1e3 2j 2.2j";
         let tokens = lex_source(source);
         assert_eq!(
             tokens,
@@ -1397,6 +1397,9 @@ mod tests {
                 Tok::Int {
                     value: BigInt::from(47),
                 },
+                Tok::Int {
+                    value: BigInt::from(10)
+                },
                 Tok::Int {
                     value: BigInt::from(13),
                 },
@@ -1410,6 +1413,8 @@ mod tests {
                     value: BigInt::from(1234567890),
                 },
                 Tok::Float { value: 0.2 },
+                Tok::Float { value: 100.0 },
+                Tok::Float { value: 2100.0 },
                 Tok::Complex {
                     real: 0.0,
                     imag: 2.0,

From 6ae42e249ba94f466259b885e40483b38494a546 Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Tue, 22 Nov 2022 21:10:19 +0900
Subject: [PATCH 298/459] Refactor lexer functions

---
 parser/src/lexer.rs | 42 +++++++++++++++++-------------------------
 1 file changed, 17 insertions(+), 25 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 62bc4ca3..2b67a56f 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -293,27 +293,26 @@ where
     /// Numeric lexing. The feast can start!
     fn lex_number(&mut self) -> LexResult {
         let start_pos = self.get_pos();
-        if self.window[0] == Some('0') {
-            if matches!(self.window[1], Some('x' | 'X')) {
+        match (self.window[0], self.window[1]) {
+            (Some('0'), Some('x' | 'X')) => {
                 // Hex! (0xdeadbeef)
                 self.next_char();
                 self.next_char();
                 self.lex_number_radix(start_pos, 16)
-            } else if matches!(self.window[1], Some('o' | 'O')) {
+            }
+            (Some('0'), Some('o' | 'O')) => {
                 // Octal style! (0o377)
                 self.next_char();
                 self.next_char();
                 self.lex_number_radix(start_pos, 8)
-            } else if matches!(self.window[1], Some('b' | 'B')) {
+            }
+            (Some('0'), Some('b' | 'B')) => {
                 // Binary! (0b_1110_0101)
                 self.next_char();
                 self.next_char();
                 self.lex_number_radix(start_pos, 2)
-            } else {
-                self.lex_normal_number()
             }
-        } else {
-            self.lex_normal_number()
+            _ => self.lex_normal_number(),
         }
     }
 
@@ -439,10 +438,9 @@ where
     fn take_number(&mut self, radix: u32) -> Option<char> {
         let take_char = Lexer::<T>::is_digit_of_radix(self.window[0], radix);
 
-        if take_char {
-            Some(self.next_char().unwrap())
-        } else {
-            None
+        match take_char {
+            true => Some(self.next_char().unwrap()),
+            _ => None,
         }
     }
 
@@ -459,12 +457,9 @@ where
 
     /// Test if we face '[eE][-+]?[0-9]+'
     fn at_exponent(&self) -> bool {
-        match self.window[0] {
-            Some('e' | 'E') => match self.window[1] {
-                Some('+' | '-') => matches!(self.window[2], Some('0'..='9')),
-                Some('0'..='9') => true,
-                _ => false,
-            },
+        match (self.window[0], self.window[1]) {
+            (Some('e' | 'E'), Some('+' | '-')) => matches!(self.window[2], Some('0'..='9')),
+            (Some('e' | 'E'), Some('0'..='9')) => true,
             _ => false,
         }
     }
@@ -705,13 +700,10 @@ where
     }
 
     fn is_identifier_continuation(&self) -> bool {
-        if let Some(c) = self.window[0] {
-            match c {
-                '_' | '0'..='9' => true,
-                c => is_xid_continue(c),
-            }
-        } else {
-            false
+        match self.window[0] {
+            Some('_' | '0'..='9') => true,
+            Some(c) => is_xid_continue(c),
+            _ => false,
         }
     }
 

From fde6b61270aae753d60bc5430f36568bced9b9de Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Wed, 23 Nov 2022 11:30:54 +0900
Subject: [PATCH 299/459] restore if expression

---
 parser/src/lexer.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 2b67a56f..8db14017 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -438,9 +438,10 @@ where
     fn take_number(&mut self, radix: u32) -> Option<char> {
         let take_char = Lexer::<T>::is_digit_of_radix(self.window[0], radix);
 
-        match take_char {
-            true => Some(self.next_char().unwrap()),
-            _ => None,
+        if take_char {
+            Some(self.next_char().unwrap())
+        } else {
+            None
         }
     }
 

From af4fb9ed527092dc171c950a73c070954ba172f3 Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Thu, 24 Nov 2022 09:23:20 +0900
Subject: [PATCH 300/459] use bool.then

---
 parser/src/lexer.rs | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 8db14017..7d24aed8 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -438,11 +438,7 @@ where
     fn take_number(&mut self, radix: u32) -> Option<char> {
         let take_char = Lexer::<T>::is_digit_of_radix(self.window[0], radix);
 
-        if take_char {
-            Some(self.next_char().unwrap())
-        } else {
-            None
-        }
+        take_char.then(|| self.next_char().unwrap())
     }
 
     /// Test if a digit is of a certain radix.

From 7f2f2cb8ad368e9441955586b0cf7afaa51f72cc Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Sun, 27 Nov 2022 23:30:52 +0900
Subject: [PATCH 301/459] Add bitflags

---
 codegen/Cargo.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml
index 3d1a455d..253d976c 100644
--- a/codegen/Cargo.toml
+++ b/codegen/Cargo.toml
@@ -12,6 +12,7 @@ rustpython-ast = { path = "../ast", features = ["unparse"] }
 rustpython-compiler-core = { path = "../core", version = "0.1.1" }
 
 ahash = "0.7.6"
+bitflags = "1.3.2"
 indexmap = "1.8.1"
 itertools = "0.10.3"
 log = "0.4.16"

From 27dd95c8f9d9ac51680398574e6301d0bac3267b Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Sun, 27 Nov 2022 23:36:25 +0900
Subject: [PATCH 302/459] Add SymbolFlags

---
 codegen/src/symboltable.rs | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index b100a9e5..24ee5ea2 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -11,6 +11,7 @@ use crate::{
     error::{CodegenError, CodegenErrorType},
     IndexMap,
 };
+use bitflags::bitflags;
 use rustpython_ast as ast;
 use rustpython_compiler_core::Location;
 use std::{borrow::Cow, fmt};
@@ -94,6 +95,33 @@ pub enum SymbolScope {
     Cell,
 }
 
+bitflags! {
+    pub struct SymbolFlags: u16 {
+        const REFERENCED = 0x001;
+        const ASSIGNED = 0x002;
+        const PARAMETER = 0x004;
+        const ANNOTATED = 0x008;
+        const IMPORTED = 0x010;
+        const NONLOCAL = 0x020;
+        // indicates if the symbol gets a value assigned by a named expression in a comprehension
+        // this is required to correct the scope in the analysis.
+        const ASSIGNED_IN_COMPREHENSION = 0x040;
+        // indicates that the symbol is used a bound iterator variable. We distinguish this case
+        // from normal assignment to detect unallowed re-assignment to iterator variables.
+        const ITER = 0x080;
+        /// indicates that the symbol is a free variable in a class method from the scope that the
+        /// class is defined in, e.g.:
+        /// ```python
+        /// def foo(x):
+        ///     class A:
+        ///         def method(self):
+        ///             return x // is_free_class
+        /// ```
+        const FREE_CLASS = 0x100;
+        const BOUND = Self::ASSIGNED.bits | Self::PARAMETER.bits | Self::IMPORTED.bits | Self::ITER.bits;
+    }
+}
+
 /// A single symbol in a table. Has various properties such as the scope
 /// of the symbol, and also the various uses of the symbol.
 #[derive(Debug, Clone)]

From eee3c03042af4933bf511bca100d225b677f581d Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Sun, 27 Nov 2022 23:43:26 +0900
Subject: [PATCH 303/459] Reduce nesting of match expression

---
 codegen/src/symboltable.rs | 59 +++++++++++++++++---------------------
 1 file changed, 27 insertions(+), 32 deletions(-)

diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 24ee5ea2..f44977d7 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -1178,38 +1178,33 @@ impl SymbolTableBuilder {
         let symbol = if let Some(symbol) = table.symbols.get_mut(name.as_ref()) {
             // Role already set..
             match role {
-                SymbolUsage::Global => {
-                    if !symbol.is_global() {
-                        if symbol.is_parameter {
-                            return Err(SymbolTableError {
-                                error: format!("name '{}' is parameter and global", name),
-                                location,
-                            });
-                        }
-                        if symbol.is_referenced {
-                            return Err(SymbolTableError {
-                                error: format!(
-                                    "name '{}' is used prior to global declaration",
-                                    name
-                                ),
-                                location,
-                            });
-                        }
-                        if symbol.is_annotated {
-                            return Err(SymbolTableError {
-                                error: format!("annotated name '{}' can't be global", name),
-                                location,
-                            });
-                        }
-                        if symbol.is_assigned {
-                            return Err(SymbolTableError {
-                                error: format!(
-                                    "name '{}' is assigned to before global declaration",
-                                    name
-                                ),
-                                location,
-                            });
-                        }
+                SymbolUsage::Global if !symbol.is_global() => {
+                    if symbol.is_parameter {
+                        return Err(SymbolTableError {
+                            error: format!("name '{}' is parameter and global", name),
+                            location,
+                        });
+                    }
+                    if symbol.is_referenced {
+                        return Err(SymbolTableError {
+                            error: format!("name '{}' is used prior to global declaration", name),
+                            location,
+                        });
+                    }
+                    if symbol.is_annotated {
+                        return Err(SymbolTableError {
+                            error: format!("annotated name '{}' can't be global", name),
+                            location,
+                        });
+                    }
+                    if symbol.is_assigned {
+                        return Err(SymbolTableError {
+                            error: format!(
+                                "name '{}' is assigned to before global declaration",
+                                name
+                            ),
+                            location,
+                        });
                     }
                 }
                 SymbolUsage::Nonlocal => {

From 7dccce5e20cff01c202c3df4067496dbbb35d5fd Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Sun, 27 Nov 2022 23:46:45 +0900
Subject: [PATCH 304/459] Fix comment

---
 codegen/src/symboltable.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index f44977d7..7bf8f50c 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -1260,7 +1260,7 @@ impl SymbolTableBuilder {
             table.symbols.entry(name.into_owned()).or_insert(symbol)
         };
 
-        // Set proper flags on symbol:
+        // Set proper scope and flags on symbol:
         match role {
             SymbolUsage::Nonlocal => {
                 symbol.scope = SymbolScope::Free;

From 7cdbb5d7b6a21b89f8934a2e97abcc654a588c12 Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Mon, 28 Nov 2022 00:46:10 +0900
Subject: [PATCH 305/459] Replace bool variable with SymbolFlags

---
 codegen/src/compile.rs     |   8 +--
 codegen/src/symboltable.rs | 100 +++++++++++++------------------------
 2 files changed, 40 insertions(+), 68 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 7bf61692..602f9b7d 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -8,7 +8,7 @@
 use crate::{
     error::{CodegenError, CodegenErrorType},
     ir,
-    symboltable::{self, SymbolScope, SymbolTable},
+    symboltable::{self, SymbolFlags, SymbolScope, SymbolTable},
     IndexSet,
 };
 use itertools::Itertools;
@@ -272,7 +272,9 @@ impl Compiler {
         let freevar_cache = table
             .symbols
             .iter()
-            .filter(|(_, s)| s.scope == SymbolScope::Free || s.is_free_class)
+            .filter(|(_, s)| {
+                s.scope == SymbolScope::Free || s.flags.contains(SymbolFlags::FREE_CLASS)
+            })
             .map(|(var, _)| var.clone())
             .collect();
 
@@ -1209,7 +1211,7 @@ impl Compiler {
             let vars = match symbol.scope {
                 SymbolScope::Free => &parent_code.freevar_cache,
                 SymbolScope::Cell => &parent_code.cellvar_cache,
-                _ if symbol.is_free_class => &parent_code.freevar_cache,
+                _ if symbol.flags.contains(SymbolFlags::FREE_CLASS) => &parent_code.freevar_cache,
                 x => unreachable!(
                     "var {} in a {:?} should be free or cell but it's {:?}",
                     var, table.typ, x
diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 7bf8f50c..7de315b8 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -129,31 +129,7 @@ pub struct Symbol {
     pub name: String,
     // pub table: SymbolTableRef,
     pub scope: SymbolScope,
-    // TODO: Use bitflags replace
-    pub is_referenced: bool,
-    pub is_assigned: bool,
-    pub is_parameter: bool,
-    pub is_annotated: bool,
-    pub is_imported: bool,
-    pub is_nonlocal: bool,
-
-    // indicates if the symbol gets a value assigned by a named expression in a comprehension
-    // this is required to correct the scope in the analysis.
-    pub is_assign_namedexpr_in_comprehension: bool,
-
-    // indicates that the symbol is used a bound iterator variable. We distinguish this case
-    // from normal assignment to detect unallowed re-assignment to iterator variables.
-    pub is_iter: bool,
-
-    /// indicates that the symbol is a free variable in a class method from the scope that the
-    /// class is defined in, e.g.:
-    /// ```python
-    /// def foo(x):
-    ///     class A:
-    ///         def method(self):
-    ///             return x // is_free_class
-    /// ```
-    pub is_free_class: bool,
+    pub flags: SymbolFlags,
 }
 
 impl Symbol {
@@ -162,15 +138,7 @@ impl Symbol {
             name: name.to_owned(),
             // table,
             scope: SymbolScope::Unknown,
-            is_referenced: false,
-            is_assigned: false,
-            is_parameter: false,
-            is_annotated: false,
-            is_imported: false,
-            is_nonlocal: false,
-            is_assign_namedexpr_in_comprehension: false,
-            is_iter: false,
-            is_free_class: false,
+            flags: SymbolFlags::empty(),
         }
     }
 
@@ -186,7 +154,7 @@ impl Symbol {
     }
 
     pub fn is_bound(&self) -> bool {
-        self.is_assigned || self.is_parameter || self.is_imported || self.is_iter
+        !(self.flags & SymbolFlags::BOUND).is_empty()
     }
 }
 
@@ -328,7 +296,11 @@ impl SymbolTableAnalyzer {
         st_typ: SymbolTableType,
         sub_tables: &mut [SymbolTable],
     ) -> SymbolTableResult {
-        if symbol.is_assign_namedexpr_in_comprehension && st_typ == SymbolTableType::Comprehension {
+        if symbol
+            .flags
+            .contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION)
+            && st_typ == SymbolTableType::Comprehension
+        {
             // propagate symbol to next higher level that can hold it,
             // i.e., function or module. Comprehension is skipped and
             // Class is not allowed and detected as error.
@@ -416,10 +388,10 @@ impl SymbolTableAnalyzer {
             for (table, typ) in self.tables.iter_mut().rev().take(decl_depth) {
                 if let SymbolTableType::Class = typ {
                     if let Some(free_class) = table.get_mut(name) {
-                        free_class.is_free_class = true;
+                        free_class.flags.insert(SymbolFlags::FREE_CLASS)
                     } else {
                         let mut symbol = Symbol::new(name);
-                        symbol.is_free_class = true;
+                        symbol.flags.insert(SymbolFlags::FREE_CLASS);
                         symbol.scope = SymbolScope::Free;
                         table.insert(name.to_owned(), symbol);
                     }
@@ -443,7 +415,7 @@ impl SymbolTableAnalyzer {
     ) -> Option<SymbolScope> {
         sub_tables.iter().find_map(|st| {
             st.symbols.get(name).and_then(|sym| {
-                if sym.scope == SymbolScope::Free || sym.is_free_class {
+                if sym.scope == SymbolScope::Free || sym.flags.contains(SymbolFlags::FREE_CLASS) {
                     if st_typ == SymbolTableType::Class && name != "__class__" {
                         None
                     } else {
@@ -474,7 +446,7 @@ impl SymbolTableAnalyzer {
         let table_type = last.1;
 
         // it is not allowed to use an iterator variable as assignee in a named expression
-        if symbol.is_iter {
+        if symbol.flags.contains(SymbolFlags::ITER) {
             return Err(SymbolTableError {
                 error: format!(
                     "assignment expression cannot rebind comprehension iteration variable {}",
@@ -501,7 +473,7 @@ impl SymbolTableAnalyzer {
                 if let Some(parent_symbol) = symbols.get_mut(&symbol.name) {
                     if let SymbolScope::Unknown = parent_symbol.scope {
                         // this information is new, as the assignment is done in inner scope
-                        parent_symbol.is_assigned = true;
+                        parent_symbol.flags.insert(SymbolFlags::ASSIGNED);
                     }
 
                     symbol.scope = if parent_symbol.is_global() {
@@ -520,7 +492,7 @@ impl SymbolTableAnalyzer {
                 match symbols.get_mut(&symbol.name) {
                     Some(parent_symbol) => {
                         // check if assignee is an iterator in top scope
-                        if parent_symbol.is_iter {
+                        if parent_symbol.flags.contains(SymbolFlags::ITER) {
                             return Err(SymbolTableError {
                                 error: format!("assignment expression cannot rebind comprehension iteration variable {}", symbol.name),
                                 // TODO: accurate location info, somehow
@@ -529,7 +501,7 @@ impl SymbolTableAnalyzer {
                         }
 
                         // we synthesize the assignment to the symbol from inner scope
-                        parent_symbol.is_assigned = true; // more checks are required
+                        parent_symbol.flags.insert(SymbolFlags::ASSIGNED); // more checks are required
                     }
                     None => {
                         // extend the scope of the inner symbol
@@ -1176,28 +1148,29 @@ impl SymbolTableBuilder {
 
         // Some checks for the symbol that present on this scope level:
         let symbol = if let Some(symbol) = table.symbols.get_mut(name.as_ref()) {
+            let flags = &symbol.flags;
             // Role already set..
             match role {
                 SymbolUsage::Global if !symbol.is_global() => {
-                    if symbol.is_parameter {
+                    if flags.contains(SymbolFlags::PARAMETER) {
                         return Err(SymbolTableError {
                             error: format!("name '{}' is parameter and global", name),
                             location,
                         });
                     }
-                    if symbol.is_referenced {
+                    if flags.contains(SymbolFlags::REFERENCED) {
                         return Err(SymbolTableError {
                             error: format!("name '{}' is used prior to global declaration", name),
                             location,
                         });
                     }
-                    if symbol.is_annotated {
+                    if flags.contains(SymbolFlags::ANNOTATED) {
                         return Err(SymbolTableError {
                             error: format!("annotated name '{}' can't be global", name),
                             location,
                         });
                     }
-                    if symbol.is_assigned {
+                    if flags.contains(SymbolFlags::ASSIGNED) {
                         return Err(SymbolTableError {
                             error: format!(
                                 "name '{}' is assigned to before global declaration",
@@ -1208,25 +1181,25 @@ impl SymbolTableBuilder {
                     }
                 }
                 SymbolUsage::Nonlocal => {
-                    if symbol.is_parameter {
+                    if flags.contains(SymbolFlags::PARAMETER) {
                         return Err(SymbolTableError {
                             error: format!("name '{}' is parameter and nonlocal", name),
                             location,
                         });
                     }
-                    if symbol.is_referenced {
+                    if flags.contains(SymbolFlags::REFERENCED) {
                         return Err(SymbolTableError {
                             error: format!("name '{}' is used prior to nonlocal declaration", name),
                             location,
                         });
                     }
-                    if symbol.is_annotated {
+                    if flags.contains(SymbolFlags::ANNOTATED) {
                         return Err(SymbolTableError {
                             error: format!("annotated name '{}' can't be nonlocal", name),
                             location,
                         });
                     }
-                    if symbol.is_assigned {
+                    if flags.contains(SymbolFlags::ASSIGNED) {
                         return Err(SymbolTableError {
                             error: format!(
                                 "name '{}' is assigned to before nonlocal declaration",
@@ -1261,47 +1234,44 @@ impl SymbolTableBuilder {
         };
 
         // Set proper scope and flags on symbol:
+        let flags = &mut symbol.flags;
         match role {
             SymbolUsage::Nonlocal => {
                 symbol.scope = SymbolScope::Free;
-                symbol.is_nonlocal = true;
+                flags.insert(SymbolFlags::NONLOCAL);
             }
             SymbolUsage::Imported => {
-                symbol.is_assigned = true;
-                symbol.is_imported = true;
+                flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::IMPORTED);
             }
             SymbolUsage::Parameter => {
-                symbol.is_parameter = true;
+                flags.insert(SymbolFlags::PARAMETER);
             }
             SymbolUsage::AnnotationParameter => {
-                symbol.is_parameter = true;
-                symbol.is_annotated = true;
+                flags.insert(SymbolFlags::PARAMETER | SymbolFlags::ANNOTATED);
             }
             SymbolUsage::AnnotationAssigned => {
-                symbol.is_assigned = true;
-                symbol.is_annotated = true;
+                flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::ANNOTATED);
             }
             SymbolUsage::Assigned => {
-                symbol.is_assigned = true;
+                flags.insert(SymbolFlags::ASSIGNED);
             }
             SymbolUsage::AssignedNamedExprInCompr => {
-                symbol.is_assigned = true;
-                symbol.is_assign_namedexpr_in_comprehension = true;
+                flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::ASSIGNED_IN_COMPREHENSION);
             }
             SymbolUsage::Global => {
                 symbol.scope = SymbolScope::GlobalExplicit;
             }
             SymbolUsage::Used => {
-                symbol.is_referenced = true;
+                flags.insert(SymbolFlags::REFERENCED);
             }
             SymbolUsage::Iter => {
-                symbol.is_iter = true;
+                flags.insert(SymbolFlags::ITER);
             }
         }
 
         // and even more checking
         // it is not allowed to assign to iterator variables (by named expressions)
-        if symbol.is_iter && symbol.is_assigned
+        if flags.contains(SymbolFlags::ITER | SymbolFlags::ASSIGNED)
         /*&& symbol.is_assign_namedexpr_in_comprehension*/
         {
             return Err(SymbolTableError {

From 7b9e7e0443114294c0e479522453a5d70d630146 Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Mon, 28 Nov 2022 20:48:03 +0900
Subject: [PATCH 306/459] Use intersects

---
 codegen/src/symboltable.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 7de315b8..9662707a 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -154,7 +154,7 @@ impl Symbol {
     }
 
     pub fn is_bound(&self) -> bool {
-        !(self.flags & SymbolFlags::BOUND).is_empty()
+        self.flags.intersects(SymbolFlags::BOUND)
     }
 }
 

From 1f7a9e5cdcd6a495894bcd05d298743d0dc12fe5 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 4 Dec 2022 05:45:15 +0900
Subject: [PATCH 307/459] Fix location

---
 parser/python.lalrpop | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index ee108edc..442a5ed8 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -756,7 +756,7 @@ LambdaDef: ast::Expr = {
 }
 
 OrTest: ast::Expr = {
-    <e1:AndTest> <location:@L> <e2:("or" AndTest)*> <end_location:@R> => {
+    <location:@L> <e1:AndTest> <e2:("or" AndTest)*> <end_location:@R> => {
         if e2.is_empty() {
             e1
         } else {
@@ -773,7 +773,7 @@ OrTest: ast::Expr = {
 };
 
 AndTest: ast::Expr = {
-    <e1:NotTest> <location:@L> <e2:("and" NotTest)*> <end_location:@R> => {
+    <location:@L> <e1:NotTest> <e2:("and" NotTest)*> <end_location:@R> => {
         if e2.is_empty() {
             e1
         } else {

From 8e0bef913c29b14915d426fe18796b51e8441dc6 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 4 Dec 2022 05:59:51 +0900
Subject: [PATCH 308/459] add tests

---
 parser/src/parser.rs                          | 14 +++++
 ...rser__parser__tests__parse_boolop_and.snap | 56 +++++++++++++++++++
 ...arser__parser__tests__parse_boolop_or.snap | 56 +++++++++++++++++++
 3 files changed, 126 insertions(+)
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_boolop_and.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__parse_boolop_or.snap

diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index d0f71fc8..f120dae7 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -218,4 +218,18 @@ class Foo(A, B):
         let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
+
+    #[test]
+    fn test_parse_boolop_or() {
+        let source = String::from("x or y");
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_boolop_and() {
+        let source = String::from("x and y");
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
 }
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_boolop_and.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_boolop_and.snap
new file mode 100644
index 00000000..f0e85620
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_boolop_and.snap
@@ -0,0 +1,56 @@
+---
+source: compiler/parser/src/parser.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 1,
+        column: 0,
+    },
+    end_location: Some(
+        Location {
+            row: 1,
+            column: 7,
+        },
+    ),
+    custom: (),
+    node: BoolOp {
+        op: And,
+        values: [
+            Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 1,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 1,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "y",
+                    ctx: Load,
+                },
+            },
+        ],
+    },
+}
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_boolop_or.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_boolop_or.snap
new file mode 100644
index 00000000..5d5abad3
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_boolop_or.snap
@@ -0,0 +1,56 @@
+---
+source: compiler/parser/src/parser.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 1,
+        column: 0,
+    },
+    end_location: Some(
+        Location {
+            row: 1,
+            column: 6,
+        },
+    ),
+    custom: (),
+    node: BoolOp {
+        op: Or,
+        values: [
+            Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 1,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 1,
+                    column: 5,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 6,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "y",
+                    ctx: Load,
+                },
+            },
+        ],
+    },
+}

From 9250260e206f467702992af8b0cf42c0742853ea Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 5 Dec 2022 12:08:16 +0900
Subject: [PATCH 309/459] Fix nightly clippy warnings

---
 codegen/src/compile.rs     |  2 +-
 codegen/src/error.rs       |  6 +++---
 codegen/src/symboltable.rs | 20 +++++++++-----------
 core/src/bytecode.rs       | 28 ++++++++++++++--------------
 parser/src/error.rs        | 21 ++++++++++-----------
 parser/src/fstring.rs      |  2 +-
 parser/src/lexer.rs        |  2 +-
 parser/src/token.rs        | 12 ++++++------
 8 files changed, 45 insertions(+), 48 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 602f9b7d..ba3d5453 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -457,7 +457,7 @@ impl Compiler {
             NameUsage::Delete if is_forbidden_name(name) => "cannot delete",
             _ => return Ok(()),
         };
-        Err(self.error(CodegenErrorType::SyntaxError(format!("{} {}", msg, name))))
+        Err(self.error(CodegenErrorType::SyntaxError(format!("{msg} {name}"))))
     }
 
     fn compile_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> {
diff --git a/codegen/src/error.rs b/codegen/src/error.rs
index 9073abb5..648446f4 100644
--- a/codegen/src/error.rs
+++ b/codegen/src/error.rs
@@ -37,8 +37,8 @@ impl fmt::Display for CodegenErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         use CodegenErrorType::*;
         match self {
-            Assign(target) => write!(f, "cannot assign to {}", target),
-            Delete(target) => write!(f, "cannot delete {}", target),
+            Assign(target) => write!(f, "cannot assign to {target}"),
+            Delete(target) => write!(f, "cannot delete {target}"),
             SyntaxError(err) => write!(f, "{}", err.as_str()),
             MultipleStarArgs => {
                 write!(f, "two starred expressions in assignment")
@@ -59,7 +59,7 @@ impl fmt::Display for CodegenErrorType {
                 "from __future__ imports must occur at the beginning of the file"
             ),
             InvalidFutureFeature(feat) => {
-                write!(f, "future feature {} is not defined", feat)
+                write!(f, "future feature {feat} is not defined")
             }
             FunctionImportStar => {
                 write!(f, "import * only allowed at module level")
diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 9662707a..5a625b25 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -1154,27 +1154,26 @@ impl SymbolTableBuilder {
                 SymbolUsage::Global if !symbol.is_global() => {
                     if flags.contains(SymbolFlags::PARAMETER) {
                         return Err(SymbolTableError {
-                            error: format!("name '{}' is parameter and global", name),
+                            error: format!("name '{name}' is parameter and global"),
                             location,
                         });
                     }
                     if flags.contains(SymbolFlags::REFERENCED) {
                         return Err(SymbolTableError {
-                            error: format!("name '{}' is used prior to global declaration", name),
+                            error: format!("name '{name}' is used prior to global declaration"),
                             location,
                         });
                     }
                     if flags.contains(SymbolFlags::ANNOTATED) {
                         return Err(SymbolTableError {
-                            error: format!("annotated name '{}' can't be global", name),
+                            error: format!("annotated name '{name}' can't be global"),
                             location,
                         });
                     }
                     if flags.contains(SymbolFlags::ASSIGNED) {
                         return Err(SymbolTableError {
                             error: format!(
-                                "name '{}' is assigned to before global declaration",
-                                name
+                                "name '{name}' is assigned to before global declaration"
                             ),
                             location,
                         });
@@ -1183,27 +1182,26 @@ impl SymbolTableBuilder {
                 SymbolUsage::Nonlocal => {
                     if flags.contains(SymbolFlags::PARAMETER) {
                         return Err(SymbolTableError {
-                            error: format!("name '{}' is parameter and nonlocal", name),
+                            error: format!("name '{name}' is parameter and nonlocal"),
                             location,
                         });
                     }
                     if flags.contains(SymbolFlags::REFERENCED) {
                         return Err(SymbolTableError {
-                            error: format!("name '{}' is used prior to nonlocal declaration", name),
+                            error: format!("name '{name}' is used prior to nonlocal declaration"),
                             location,
                         });
                     }
                     if flags.contains(SymbolFlags::ANNOTATED) {
                         return Err(SymbolTableError {
-                            error: format!("annotated name '{}' can't be nonlocal", name),
+                            error: format!("annotated name '{name}' can't be nonlocal"),
                             location,
                         });
                     }
                     if flags.contains(SymbolFlags::ASSIGNED) {
                         return Err(SymbolTableError {
                             error: format!(
-                                "name '{}' is assigned to before nonlocal declaration",
-                                name
+                                "name '{name}' is assigned to before nonlocal declaration"
                             ),
                             location,
                         });
@@ -1220,7 +1218,7 @@ impl SymbolTableBuilder {
             match role {
                 SymbolUsage::Nonlocal if scope_depth < 2 => {
                     return Err(SymbolTableError {
-                        error: format!("cannot define nonlocal '{}' at top level.", name),
+                        error: format!("cannot define nonlocal '{name}' at top level."),
                         location,
                     })
                 }
diff --git a/core/src/bytecode.rs b/core/src/bytecode.rs
index 9c3751cd..ea2477ae 100644
--- a/core/src/bytecode.rs
+++ b/core/src/bytecode.rs
@@ -482,15 +482,15 @@ impl<C: Constant> BorrowedConstant<'_, C> {
     // takes `self` because we need to consume the iterator
     pub fn fmt_display(self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            BorrowedConstant::Integer { value } => write!(f, "{}", value),
-            BorrowedConstant::Float { value } => write!(f, "{}", value),
-            BorrowedConstant::Complex { value } => write!(f, "{}", value),
+            BorrowedConstant::Integer { value } => write!(f, "{value}"),
+            BorrowedConstant::Float { value } => write!(f, "{value}"),
+            BorrowedConstant::Complex { value } => write!(f, "{value}"),
             BorrowedConstant::Boolean { value } => {
                 write!(f, "{}", if value { "True" } else { "False" })
             }
-            BorrowedConstant::Str { value } => write!(f, "{:?}", value),
+            BorrowedConstant::Str { value } => write!(f, "{value:?}"),
             BorrowedConstant::Bytes { value } => write!(f, "b{:?}", value.as_bstr()),
-            BorrowedConstant::Code { code } => write!(f, "{:?}", code),
+            BorrowedConstant::Code { code } => write!(f, "{code:?}"),
             BorrowedConstant::Tuple { elements } => {
                 write!(f, "(")?;
                 let mut first = true;
@@ -852,7 +852,7 @@ impl<C: Constant> fmt::Display for CodeObject<C> {
         self.display_inner(f, false, 1)?;
         for constant in &*self.constants {
             if let BorrowedConstant::Code { code } = constant.borrow_constant() {
-                writeln!(f, "\nDisassembly of {:?}", code)?;
+                writeln!(f, "\nDisassembly of {code:?}")?;
                 code.fmt(f)?;
             }
         }
@@ -1118,14 +1118,14 @@ impl Instruction {
                     }
                 }
             }
-            UnaryOperation { op } => w!(UnaryOperation, format_args!("{:?}", op)),
-            BinaryOperation { op } => w!(BinaryOperation, format_args!("{:?}", op)),
+            UnaryOperation { op } => w!(UnaryOperation, format_args!("{op:?}")),
+            BinaryOperation { op } => w!(BinaryOperation, format_args!("{op:?}")),
             BinaryOperationInplace { op } => {
-                w!(BinaryOperationInplace, format_args!("{:?}", op))
+                w!(BinaryOperationInplace, format_args!("{op:?}"))
             }
             LoadAttr { idx } => w!(LoadAttr, name(*idx)),
-            TestOperation { op } => w!(TestOperation, format_args!("{:?}", op)),
-            CompareOperation { op } => w!(CompareOperation, format_args!("{:?}", op)),
+            TestOperation { op } => w!(TestOperation, format_args!("{op:?}")),
+            CompareOperation { op } => w!(CompareOperation, format_args!("{op:?}")),
             Pop => w!(Pop),
             Rotate2 => w!(Rotate2),
             Rotate3 => w!(Rotate3),
@@ -1139,7 +1139,7 @@ impl Instruction {
             JumpIfFalse { target } => w!(JumpIfFalse, target),
             JumpIfTrueOrPop { target } => w!(JumpIfTrueOrPop, target),
             JumpIfFalseOrPop { target } => w!(JumpIfFalseOrPop, target),
-            MakeFunction(flags) => w!(MakeFunction, format_args!("{:?}", flags)),
+            MakeFunction(flags) => w!(MakeFunction, format_args!("{flags:?}")),
             CallFunctionPositional { nargs } => w!(CallFunctionPositional, nargs),
             CallFunctionKeyword { nargs } => w!(CallFunctionKeyword, nargs),
             CallFunctionEx { has_kwargs } => w!(CallFunctionEx, has_kwargs),
@@ -1163,7 +1163,7 @@ impl Instruction {
             BeforeAsyncWith => w!(BeforeAsyncWith),
             SetupAsyncWith { end } => w!(SetupAsyncWith, end),
             PopBlock => w!(PopBlock),
-            Raise { kind } => w!(Raise, format_args!("{:?}", kind)),
+            Raise { kind } => w!(Raise, format_args!("{kind:?}")),
             BuildString { size } => w!(BuildString, size),
             BuildTuple { size, unpack } => w!(BuildTuple, size, unpack),
             BuildList { size, unpack } => w!(BuildList, size, unpack),
@@ -1182,7 +1182,7 @@ impl Instruction {
             LoadBuildClass => w!(LoadBuildClass),
             UnpackSequence { size } => w!(UnpackSequence, size),
             UnpackEx { before, after } => w!(UnpackEx, before, after),
-            FormatValue { conversion } => w!(FormatValue, format_args!("{:?}", conversion)),
+            FormatValue { conversion } => w!(FormatValue, format_args!("{conversion:?}")),
             PopException => w!(PopException),
             Reverse { amount } => w!(Reverse, amount),
             GetAwaitable => w!(GetAwaitable),
diff --git a/parser/src/error.rs b/parser/src/error.rs
index e8354421..89f53668 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -34,7 +34,7 @@ impl fmt::Display for LexicalErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             LexicalErrorType::StringError => write!(f, "Got unexpected string"),
-            LexicalErrorType::FStringError(error) => write!(f, "Got error in f-string: {}", error),
+            LexicalErrorType::FStringError(error) => write!(f, "Got error in f-string: {error}"),
             LexicalErrorType::UnicodeError => write!(f, "Got unexpected unicode"),
             LexicalErrorType::NestingError => write!(f, "Got unexpected nesting"),
             LexicalErrorType::IndentationError => {
@@ -56,13 +56,13 @@ impl fmt::Display for LexicalErrorType {
                 write!(f, "positional argument follows keyword argument")
             }
             LexicalErrorType::UnrecognizedToken { tok } => {
-                write!(f, "Got unexpected token {}", tok)
+                write!(f, "Got unexpected token {tok}")
             }
             LexicalErrorType::LineContinuationError => {
                 write!(f, "unexpected character after line continuation character")
             }
             LexicalErrorType::Eof => write!(f, "unexpected EOF while parsing"),
-            LexicalErrorType::OtherError(msg) => write!(f, "{}", msg),
+            LexicalErrorType::OtherError(msg) => write!(f, "{msg}"),
         }
     }
 }
@@ -97,17 +97,16 @@ impl fmt::Display for FStringErrorType {
             FStringErrorType::UnopenedRbrace => write!(f, "Unopened '}}'"),
             FStringErrorType::ExpectedRbrace => write!(f, "Expected '}}' after conversion flag."),
             FStringErrorType::InvalidExpression(error) => {
-                write!(f, "{}", error)
+                write!(f, "{error}")
             }
             FStringErrorType::InvalidConversionFlag => write!(f, "invalid conversion character"),
             FStringErrorType::EmptyExpression => write!(f, "empty expression not allowed"),
             FStringErrorType::MismatchedDelimiter(first, second) => write!(
                 f,
-                "closing parenthesis '{}' does not match opening parenthesis '{}'",
-                second, first
+                "closing parenthesis '{second}' does not match opening parenthesis '{first}'"
             ),
             FStringErrorType::SingleRbrace => write!(f, "single '}}' is not allowed"),
-            FStringErrorType::Unmatched(delim) => write!(f, "unmatched '{}'", delim),
+            FStringErrorType::Unmatched(delim) => write!(f, "unmatched '{delim}'"),
             FStringErrorType::ExpressionNestedTooDeeply => {
                 write!(f, "expressions nested too deeply")
             }
@@ -118,7 +117,7 @@ impl fmt::Display for FStringErrorType {
                 if *c == '\\' {
                     write!(f, "f-string expression part cannot include a backslash")
                 } else {
-                    write!(f, "f-string expression part cannot include '{}'s", c)
+                    write!(f, "f-string expression part cannot include '{c}'s")
                 }
             }
         }
@@ -198,7 +197,7 @@ impl fmt::Display for ParseErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match *self {
             ParseErrorType::Eof => write!(f, "Got unexpected EOF"),
-            ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {:?}", tok),
+            ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {tok:?}"),
             ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
             ParseErrorType::UnrecognizedToken(ref tok, ref expected) => {
                 if *tok == Tok::Indent {
@@ -206,10 +205,10 @@ impl fmt::Display for ParseErrorType {
                 } else if expected.as_deref() == Some("Indent") {
                     write!(f, "expected an indented block")
                 } else {
-                    write!(f, "invalid syntax. Got unexpected token {}", tok)
+                    write!(f, "invalid syntax. Got unexpected token {tok}")
                 }
             }
-            ParseErrorType::Lexical(ref error) => write!(f, "{}", error),
+            ParseErrorType::Lexical(ref error) => write!(f, "{error}"),
         }
     }
 }
diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index 64cb7d3b..a28383f1 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -344,7 +344,7 @@ impl FStringParser {
 }
 
 fn parse_fstring_expr(source: &str) -> Result<Expr, ParseError> {
-    let fstring_body = format!("({})", source);
+    let fstring_body = format!("({source})");
     parse_expression(&fstring_body, "<fstring>")
 }
 
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 7d24aed8..04140aa8 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -321,7 +321,7 @@ where
         let value_text = self.radix_run(radix);
         let end_pos = self.get_pos();
         let value = BigInt::from_str_radix(&value_text, radix).map_err(|e| LexicalError {
-            error: LexicalErrorType::OtherError(format!("{:?}", e)),
+            error: LexicalErrorType::OtherError(format!("{e:?}")),
             location: start_pos,
         })?;
         Ok((start_pos, Tok::Int { value }, end_pos))
diff --git a/parser/src/token.rs b/parser/src/token.rs
index ba76f6a4..14ffece5 100644
--- a/parser/src/token.rs
+++ b/parser/src/token.rs
@@ -118,17 +118,17 @@ impl fmt::Display for Tok {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         use Tok::*;
         match self {
-            Name { name } => write!(f, "'{}'", name),
-            Int { value } => write!(f, "'{}'", value),
-            Float { value } => write!(f, "'{}'", value),
-            Complex { real, imag } => write!(f, "{}j{}", real, imag),
+            Name { name } => write!(f, "'{name}'"),
+            Int { value } => write!(f, "'{value}'"),
+            Float { value } => write!(f, "'{value}'"),
+            Complex { real, imag } => write!(f, "{real}j{imag}"),
             String { value, kind } => {
                 match kind {
                     StringKind::F => f.write_str("f")?,
                     StringKind::U => f.write_str("u")?,
                     StringKind::Normal => {}
                 }
-                write!(f, "{:?}", value)
+                write!(f, "{value:?}")
             }
             Bytes { value } => {
                 write!(f, "b\"")?;
@@ -138,7 +138,7 @@ impl fmt::Display for Tok {
                         10 => f.write_str("\\n")?,
                         13 => f.write_str("\\r")?,
                         32..=126 => f.write_char(*i as char)?,
-                        _ => write!(f, "\\x{:02x}", i)?,
+                        _ => write!(f, "\\x{i:02x}")?,
                     }
                 }
                 f.write_str("\"")

From dd01ab1c9e94039bb08c307c8a781cdbd71a3d48 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sat, 10 Dec 2022 17:49:57 +0900
Subject: [PATCH 310/459] Fix the end location of an implicitly-concatenated
 string

---
 ...tpython_parser__string__tests__parse_f_string_concat_1.snap | 2 +-
 ...tpython_parser__string__tests__parse_f_string_concat_2.snap | 2 +-
 ...tpython_parser__string__tests__parse_f_string_concat_3.snap | 2 +-
 ...ython_parser__string__tests__parse_u_f_string_concat_1.snap | 2 +-
 ...ython_parser__string__tests__parse_u_f_string_concat_2.snap | 2 +-
 parser/src/string.rs                                           | 3 ++-
 6 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
index f57f903b..9b1f5668 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
@@ -24,7 +24,7 @@ expression: parse_ast
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 8,
+                        column: 17,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
index f57f903b..9b1f5668 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
@@ -24,7 +24,7 @@ expression: parse_ast
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 8,
+                        column: 17,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
index 68582508..0d7ebcab 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
@@ -24,7 +24,7 @@ expression: parse_ast
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 8,
+                        column: 22,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
index d9d57865..2449dff3 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
@@ -24,7 +24,7 @@ expression: parse_ast
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 9,
+                        column: 18,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
index a9f10f06..f56ae0f2 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
@@ -24,7 +24,7 @@ expression: parse_ast
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 9,
+                        column: 22,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/string.rs b/parser/src/string.rs
index 838da94c..5a8e3e6e 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -12,6 +12,7 @@ pub fn parse_strings(
     // Preserve the initial location and kind.
     let initial_start = values[0].0;
     let initial_end = values[0].2;
+    let last_end = values.last().unwrap().2;
     let initial_kind = (values[0].1 .1 == StringKind::U).then(|| "u".to_owned());
 
     // Optimization: fast-track the common case of a single string.
@@ -84,7 +85,7 @@ pub fn parse_strings(
     Ok(if has_fstring {
         Expr::new(
             initial_start,
-            initial_end,
+            last_end,
             ExprKind::JoinedStr { values: deduped },
         )
     } else {

From 9111cd8a3d12c9808efc4402e98e267a19be04ab Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sat, 10 Dec 2022 18:09:26 +0900
Subject: [PATCH 311/459] Fix plain string

---
 ...n_parser__string__tests__parse_string_concat.snap |  2 +-
 ...rser__string__tests__parse_u_string_concat_1.snap |  2 +-
 ...rser__string__tests__parse_u_string_concat_2.snap |  2 +-
 parser/src/string.rs                                 | 12 +++++-------
 4 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
index 0052f8df..a8744263 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap
@@ -24,7 +24,7 @@ expression: parse_ast
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 8,
+                        column: 16,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
index 503d5b5c..868ba813 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap
@@ -24,7 +24,7 @@ expression: parse_ast
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 8,
+                        column: 17,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
index 43262599..a24fc2b3 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap
@@ -24,7 +24,7 @@ expression: parse_ast
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 9,
+                        column: 17,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/string.rs b/parser/src/string.rs
index 5a8e3e6e..3d0a4525 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -82,18 +82,16 @@ pub fn parse_strings(
         deduped.push(take_current(&mut current));
     }
 
-    Ok(if has_fstring {
-        Expr::new(
-            initial_start,
-            last_end,
-            ExprKind::JoinedStr { values: deduped },
-        )
+    let node = if has_fstring {
+        ExprKind::JoinedStr { values: deduped }
     } else {
         deduped
             .into_iter()
             .exactly_one()
             .expect("String must be concatenated to a single element.")
-    })
+            .node
+    };
+    Ok(Expr::new(initial_start, last_end, node))
 }
 
 #[cfg(test)]

From d45ac01ebe86d21da4b48c8cd205d86a4b4cfc15 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sat, 10 Dec 2022 18:45:36 +0900
Subject: [PATCH 312/459] Fix

---
 ...ython_parser__string__tests__parse_f_string_concat_1.snap | 2 +-
 ...ython_parser__string__tests__parse_f_string_concat_2.snap | 2 +-
 ...ython_parser__string__tests__parse_f_string_concat_3.snap | 2 +-
 ...hon_parser__string__tests__parse_u_f_string_concat_1.snap | 2 +-
 ...hon_parser__string__tests__parse_u_f_string_concat_2.snap | 2 +-
 parser/src/string.rs                                         | 5 ++---
 6 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
index 9b1f5668..bfe109e1 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap
@@ -38,7 +38,7 @@ expression: parse_ast
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 8,
+                                    column: 17,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
index 9b1f5668..bfe109e1 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap
@@ -38,7 +38,7 @@ expression: parse_ast
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 8,
+                                    column: 17,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
index 0d7ebcab..87b92127 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
@@ -38,7 +38,7 @@ expression: parse_ast
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 8,
+                                    column: 22,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
index 2449dff3..76e6b1d2 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap
@@ -38,7 +38,7 @@ expression: parse_ast
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 9,
+                                    column: 18,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
index f56ae0f2..8c073fd9 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap
@@ -38,7 +38,7 @@ expression: parse_ast
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 9,
+                                    column: 22,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/string.rs b/parser/src/string.rs
index 3d0a4525..cf18de9c 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -11,7 +11,6 @@ pub fn parse_strings(
 ) -> Result<Expr, LexicalError> {
     // Preserve the initial location and kind.
     let initial_start = values[0].0;
-    let initial_end = values[0].2;
     let last_end = values.last().unwrap().2;
     let initial_kind = (values[0].1 .1 == StringKind::U).then(|| "u".to_owned());
 
@@ -20,7 +19,7 @@ pub fn parse_strings(
         let value = values.into_iter().last().unwrap().1 .0;
         return Ok(Expr::new(
             initial_start,
-            initial_end,
+            last_end,
             ExprKind::Constant {
                 value: Constant::Str(value),
                 kind: initial_kind,
@@ -39,7 +38,7 @@ pub fn parse_strings(
     let take_current = |current: &mut Vec<String>| -> Expr {
         Expr::new(
             initial_start,
-            initial_end,
+            last_end,
             ExprKind::Constant {
                 value: Constant::Str(current.drain(..).join("")),
                 kind: initial_kind.clone(),

From 2b91ffb3ae03ff53da23036e5d6da7f39c7cff86 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sat, 10 Dec 2022 22:01:42 +0900
Subject: [PATCH 313/459] Refactor

---
 parser/src/string.rs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/parser/src/string.rs b/parser/src/string.rs
index cf18de9c..b80c6298 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -81,16 +81,18 @@ pub fn parse_strings(
         deduped.push(take_current(&mut current));
     }
 
-    let node = if has_fstring {
-        ExprKind::JoinedStr { values: deduped }
+    Ok(if has_fstring {
+        Expr::new(
+            initial_start,
+            last_end,
+            ExprKind::JoinedStr { values: deduped },
+        )
     } else {
         deduped
             .into_iter()
             .exactly_one()
             .expect("String must be concatenated to a single element.")
-            .node
-    };
-    Ok(Expr::new(initial_start, last_end, node))
+    })
 }
 
 #[cfg(test)]

From 2d75aeb2763ee9dbeedd8218d8ef394d181eb881 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 11 Dec 2022 12:35:28 +0900
Subject: [PATCH 314/459] Fix end location of nodes containing body

---
 parser/python.lalrpop                                     | 8 +++++---
 parser/src/parser.rs                                      | 3 ++-
 .../rustpython_parser__parser__tests__parse_class.snap    | 4 ++--
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 442a5ed8..b4f85b51 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -492,9 +492,10 @@ WithItem: ast::Withitem = {
 };
 
 FuncDef: ast::Stmt = {
-    <decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test)?> ":" <body:Suite> <end_location:@R>  => {
+    <decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test)?> ":" <body:Suite>  => {
         let args = Box::new(args);
         let returns = r.map(|x| Box::new(x.1));
+        let end_location = body.last().unwrap().end_location.unwrap();
         let type_comment = None;
         let node = if is_async.is_some() {
             ast::StmtKind::AsyncFunctionDef { name, args, body, decorator_list, returns, type_comment }
@@ -646,15 +647,16 @@ KwargParameter<ArgType>: Option<Box<ast::Arg>> = {
 };
 
 ClassDef: ast::Stmt = {
-    <decorator_list:Decorator*> <location:@L> "class" <name:Identifier> <a:("(" ArgumentList ")")?> ":" <body:Suite> <end_location:@R> => {
+    <decorator_list:Decorator*> <location:@L> "class" <name:Identifier> <a:("(" ArgumentList ")")?> ":" <body:Suite> => {
         let (bases, keywords) = match a {
             Some((_, arg, _)) => (arg.args, arg.keywords),
             None => (vec![], vec![]),
         };
+        let end_location = body.last().unwrap().end_location;
         ast::Stmt {
             custom: (),
             location,
-            end_location: Some(end_location),
+            end_location,
             node: ast::StmtKind::ClassDef {
                 name,
                 bases,
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index f120dae7..27192b6a 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -173,7 +173,8 @@ class Foo(A, B):
  def __init__(self):
   pass
  def method_with_default(self, arg='default'):
-  pass";
+  pass
+";
         insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
     }
 
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
index c74d2eb9..22b15410 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap
@@ -62,8 +62,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                     },
                     end_location: Some(
                         Location {
-                            row: 4,
-                            column: 1,
+                            row: 3,
+                            column: 6,
                         },
                     ),
                     custom: (),

From c79cc34b377ff92b0c5dad7eadb361c38b4b614f Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Mon, 12 Dec 2022 01:10:42 +0900
Subject: [PATCH 315/459] Fix other compound statements

---
 parser/python.lalrpop                         | 57 ++++++++++++++-----
 ...er__parser__tests__parse_if_elif_else.snap |  4 +-
 2 files changed, 46 insertions(+), 15 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index b4f85b51..205bee0c 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -357,16 +357,24 @@ CompoundStatement: ast::Stmt = {
 };
 
 IfStatement: ast::Stmt = {
-    <location:@L> "if" <test:NamedExpressionTest> ":" <body:Suite> <s2:(@L "elif" NamedExpressionTest ":" Suite @R)*> <s3:("else" ":" Suite)?> <end_location:@R> => {
+    <location:@L> "if" <test:NamedExpressionTest> ":" <body:Suite> <s2:(@L "elif" NamedExpressionTest ":" Suite)*> <s3:("else" ":" Suite)?> => {
         // Determine last else:
         let mut last = s3.map(|s| s.2).unwrap_or_default();
-
+        let end_location = if let Some(last) = last.last() {
+            last.end_location
+        } else {
+            if let Some(last) = s2.last() {
+                last.4.last().unwrap().end_location
+            } else {
+                body.last().unwrap().end_location
+            }
+        };
         // handle elif:
         for i in s2.into_iter().rev() {
             let x = ast::Stmt {
                 custom: (),
                 location: i.0,
-                end_location: Some(i.5),
+                end_location: i.4.last().unwrap().end_location,
                 node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last },
             };
             last = vec![x];
@@ -375,19 +383,24 @@ IfStatement: ast::Stmt = {
         ast::Stmt {
             custom: (),
             location,
-            end_location: Some(end_location),
+            end_location,
             node: ast::StmtKind::If { test: Box::new(test), body, orelse: last }
         }
     },
 };
 
 WhileStatement: ast::Stmt = {
-    <location:@L> "while" <test:NamedExpressionTest> ":" <body:Suite> <s2:("else" ":" Suite)?> <end_location:@R> => {
+    <location:@L> "while" <test:NamedExpressionTest> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
+        let end_location = if let Some(ref s) = s2 {
+            s.2.last().unwrap().end_location
+        } else {
+            body.last().unwrap().end_location
+        };
         let orelse = s2.map(|s| s.2).unwrap_or_default();
         ast::Stmt {
             custom: (),
             location,
-            end_location: Some(end_location),
+            end_location,
             node: ast::StmtKind::While {
                 test: Box::new(test),
                 body,
@@ -398,7 +411,12 @@ WhileStatement: ast::Stmt = {
 };
 
 ForStatement: ast::Stmt = {
-    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> <end_location:@R> => {
+    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
+        let end_location = if let Some(ref s) = s2 {
+            s.2.last().unwrap().end_location.unwrap()
+        } else {
+            body.last().unwrap().end_location.unwrap()
+        };
         let orelse = s2.map(|s| s.2).unwrap_or_default();
         let target = Box::new(set_context(target, ast::ExprContext::Store));
         let iter = Box::new(iter);
@@ -416,10 +434,19 @@ TryStatement: ast::Stmt = {
     <location:@L> "try" ":" <body:Suite> <handlers:ExceptClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> <end_location:@R> => {
         let orelse = else_suite.map(|s| s.2).unwrap_or_default();
         let finalbody = finally.map(|s| s.2).unwrap_or_default();
+        let end_location = if let Some(last) = finalbody.last() {
+            last.end_location
+        } else {
+            if let Some(last) = orelse.last() {
+                last.end_location
+            } else {
+                handlers.last().unwrap().end_location
+            }
+        };
         ast::Stmt {
             custom: (),
             location,
-            end_location: Some(end_location),
+            end_location,
             node: ast::StmtKind::Try {
                 body,
                 handlers,
@@ -428,14 +455,15 @@ TryStatement: ast::Stmt = {
             },
         }
     },
-    <location:@L> "try" ":" <body:Suite> <finally:("finally" ":" Suite)> <end_location:@R> => {
+    <location:@L> "try" ":" <body:Suite> <finally:("finally" ":" Suite)> => {
         let handlers = vec![];
         let orelse = vec![];
         let finalbody = finally.2;
+        let end_location = finalbody.last().unwrap().end_location;
         ast::Stmt {
             custom: (),
             location,
-            end_location: Some(end_location),
+            end_location,
             node: ast::StmtKind::Try {
                 body,
                 handlers,
@@ -447,7 +475,8 @@ TryStatement: ast::Stmt = {
 };
 
 ExceptClause: ast::Excepthandler = {
-    <location:@L> "except" <typ:Test?> ":" <body:Suite> <end_location:@R> => {
+    <location:@L> "except" <typ:Test?> ":" <body:Suite> => {
+        let end_location = body.last().unwrap().end_location.unwrap();
         ast::Excepthandler::new(
             location,
             end_location,
@@ -458,7 +487,8 @@ ExceptClause: ast::Excepthandler = {
             },
         )
     },
-    <location:@L> "except" <x:(Test "as" Identifier)> ":" <body:Suite> <end_location:@R> => {
+    <location:@L> "except" <x:(Test "as" Identifier)> ":" <body:Suite> => {
+        let end_location = body.last().unwrap().end_location.unwrap();
         ast::Excepthandler::new(
             location,
             end_location,
@@ -472,7 +502,8 @@ ExceptClause: ast::Excepthandler = {
 };
 
 WithStatement: ast::Stmt = {
-    <location:@L> <is_async:"async"?> "with" <items:OneOrMore<WithItem>> ":" <body:Suite> <end_location:@R> => {
+    <location:@L> <is_async:"async"?> "with" <items:OneOrMore<WithItem>> ":" <body:Suite> => {
+        let end_location = body.last().unwrap().end_location.unwrap();
         let type_comment = None;
         let node = if is_async.is_some() {
             ast::StmtKind::AsyncWith { items, body, type_comment }
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
index e65226d6..7f493ddd 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
@@ -79,8 +79,8 @@ expression: parse_ast
                     },
                     end_location: Some(
                         Location {
-                            row: 3,
-                            column: 0,
+                            row: 2,
+                            column: 10,
                         },
                     ),
                     custom: (),

From cfbf5f0f6302eed4b0653f46f1839c5c54232e4f Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Mon, 12 Dec 2022 10:24:00 +0900
Subject: [PATCH 316/459] Use method chaining

---
 parser/python.lalrpop | 47 +++++++++++++++++--------------------------
 1 file changed, 19 insertions(+), 28 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 205bee0c..ff33fe4f 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -360,15 +360,11 @@ IfStatement: ast::Stmt = {
     <location:@L> "if" <test:NamedExpressionTest> ":" <body:Suite> <s2:(@L "elif" NamedExpressionTest ":" Suite)*> <s3:("else" ":" Suite)?> => {
         // Determine last else:
         let mut last = s3.map(|s| s.2).unwrap_or_default();
-        let end_location = if let Some(last) = last.last() {
-            last.end_location
-        } else {
-            if let Some(last) = s2.last() {
-                last.4.last().unwrap().end_location
-            } else {
-                body.last().unwrap().end_location
-            }
-        };
+        let end_location = last
+            .last()
+            .map(|last| last.end_location)
+            .or_else(|| s2.last().map(|last| last.4.last().unwrap().end_location))
+            .unwrap_or_else(|| body.last().unwrap().end_location);
         // handle elif:
         for i in s2.into_iter().rev() {
             let x = ast::Stmt {
@@ -391,11 +387,10 @@ IfStatement: ast::Stmt = {
 
 WhileStatement: ast::Stmt = {
     <location:@L> "while" <test:NamedExpressionTest> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
-        let end_location = if let Some(ref s) = s2 {
-            s.2.last().unwrap().end_location
-        } else {
-            body.last().unwrap().end_location
-        };
+        let end_location = s2
+            .as_ref()
+            .map(|s| s.2.last().unwrap().end_location)
+            .unwrap_or_else(|| body.last().unwrap().end_location);
         let orelse = s2.map(|s| s.2).unwrap_or_default();
         ast::Stmt {
             custom: (),
@@ -412,11 +407,11 @@ WhileStatement: ast::Stmt = {
 
 ForStatement: ast::Stmt = {
     <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
-        let end_location = if let Some(ref s) = s2 {
-            s.2.last().unwrap().end_location.unwrap()
-        } else {
-            body.last().unwrap().end_location.unwrap()
-        };
+        let end_location = s2
+            .as_ref()
+            .map(|s| s.2.last().unwrap().end_location)
+            .unwrap_or_else(|| body.last().unwrap().end_location)
+            .unwrap();
         let orelse = s2.map(|s| s.2).unwrap_or_default();
         let target = Box::new(set_context(target, ast::ExprContext::Store));
         let iter = Box::new(iter);
@@ -434,15 +429,11 @@ TryStatement: ast::Stmt = {
     <location:@L> "try" ":" <body:Suite> <handlers:ExceptClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> <end_location:@R> => {
         let orelse = else_suite.map(|s| s.2).unwrap_or_default();
         let finalbody = finally.map(|s| s.2).unwrap_or_default();
-        let end_location = if let Some(last) = finalbody.last() {
-            last.end_location
-        } else {
-            if let Some(last) = orelse.last() {
-                last.end_location
-            } else {
-                handlers.last().unwrap().end_location
-            }
-        };
+        let end_location = finalbody
+            .last()
+            .map(|last| last.end_location)
+            .or_else(|| orelse.last().map(|last| last.end_location))
+            .unwrap_or_else(|| handlers.last().unwrap().end_location);
         ast::Stmt {
             custom: (),
             location,

From dec0bf571f9bb41d94dccaaadeebb041a65efade Mon Sep 17 00:00:00 2001
From: Anders Kaseorg <andersk@mit.edu>
Date: Sun, 11 Dec 2022 03:16:48 -0800
Subject: [PATCH 317/459] Split and simplify some LALRPOP rules

Signed-off-by: Anders Kaseorg <andersk@mit.edu>
---
 parser/python.lalrpop | 138 +++++++++++++++++-------------------------
 1 file changed, 56 insertions(+), 82 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 442a5ed8..565130cb 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -689,22 +689,17 @@ YieldExpr: ast::Expr = {
 };
 
 Test: ast::Expr = {
-    <expr:OrTest> <condition: (@L "if" OrTest "else" Test @R)?> => {
-        if let Some(c) = condition {
-            ast::Expr {
-                location: c.0,
-                end_location: Some(c.5),
-                custom: (),
-                node: ast::ExprKind::IfExp {
-                    test: Box::new(c.2),
-                    body: Box::new(expr),
-                    orelse: Box::new(c.4),
-                }
-            }
-        } else {
-            expr
+    <body:OrTest> <location:@L> "if" <test:OrTest> "else" <orelse:Test> <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::IfExp {
+            test: Box::new(test),
+            body: Box::new(body),
+            orelse: Box::new(orelse),
         }
     },
+    OrTest,
     LambdaDef,
 };
 
@@ -756,37 +751,31 @@ LambdaDef: ast::Expr = {
 }
 
 OrTest: ast::Expr = {
-    <location:@L> <e1:AndTest> <e2:("or" AndTest)*> <end_location:@R> => {
-        if e2.is_empty() {
-            e1
-        } else {
-            let mut values = vec![e1];
-            values.extend(e2.into_iter().map(|e| e.1));
-            ast::Expr {
-                location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values }
-            }
+    <location:@L> <e1:AndTest> <e2:("or" AndTest)+> <end_location:@R> => {
+        let mut values = vec![e1];
+        values.extend(e2.into_iter().map(|e| e.1));
+        ast::Expr {
+            location,
+            end_location: Some(end_location),
+            custom: (),
+            node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values }
         }
     },
+    AndTest,
 };
 
 AndTest: ast::Expr = {
-    <location:@L> <e1:NotTest> <e2:("and" NotTest)*> <end_location:@R> => {
-        if e2.is_empty() {
-            e1
-        } else {
-            let mut values = vec![e1];
-            values.extend(e2.into_iter().map(|e| e.1));
-            ast::Expr {
-                location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values }
-            }
+    <location:@L> <e1:NotTest> <e2:("and" NotTest)+> <end_location:@R> => {
+        let mut values = vec![e1];
+        values.extend(e2.into_iter().map(|e| e.1));
+        ast::Expr {
+            location,
+            end_location: Some(end_location),
+            custom: (),
+            node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values }
         }
     },
+    NotTest,
 };
 
 NotTest: ast::Expr = {
@@ -920,32 +909,23 @@ UnaryOp: ast::Unaryop = {
 };
 
 Power: ast::Expr = {
-    <e:AtomExpr> <e2:(@L "**" Factor @R)?> => {
-        match e2 {
-            None => e,
-            Some((location, _, b, end_location)) => ast::Expr {
-                location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) }
-            },
-        }
-    }
+    <e:AtomExpr> <location:@L> "**" <b:Factor> <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) }
+    },
+    AtomExpr,
 };
 
 AtomExpr: ast::Expr = {
-    <location:@L> <is_await:"await"?> <atom:AtomExpr2> <end_location:@R> => {
-        if is_await.is_some() {
-            ast::Expr {
-                location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::ExprKind::Await { value: Box::new(atom) }
-            }
-        } else {
-            atom
-        }
-    }
+    <location:@L> "await" <atom:AtomExpr2> <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::Await { value: Box::new(atom) }
+    },
+    AtomExpr2,
 }
 
 AtomExpr2: ast::Expr = {
@@ -1042,30 +1022,24 @@ Atom: ast::Expr = {
             node: ast::ExprKind::ListComp { elt: Box::new(elt), generators }
         }
     },
-    <location:@L> "(" <elements:TestOrStarNamedExprList?> ")" <end_location:@R> =>? {
-        match elements {
-            Some(elt) => {
-                match elt.node {
-                    ast::ExprKind::Starred { .. } => {
-                        Err(LexicalError{
-                            error : LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
-                            location: location,
-                        }.into())
-                    },
-                    _ => {
-                        Ok(elt)
-                    }
-                }
-            },
-            None => {
-                Ok(ast::Expr::new(
-                     location,
-                     end_location,
-                     ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load }
-                ))
+    "(" <elt:TestOrStarNamedExprList> ")" =>? {
+        match elt.node {
+            ast::ExprKind::Starred { .. } => {
+                Err(LexicalError{
+                    error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
+                    location: elt.location,
+                }.into())
+            }
+            _ => {
+                Ok(elt)
             }
         }
     },
+    <location:@L> "(" ")" <end_location:@R> => ast::Expr::new(
+        location,
+        end_location,
+        ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load }
+    ),
     "(" <e:YieldExpr> ")" => e,
     <location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" <end_location:@R> => {
         ast::Expr {

From bfd847d04cd240bc1fdf7ccc5536a3db60ae20d9 Mon Sep 17 00:00:00 2001
From: Anders Kaseorg <andersk@mit.edu>
Date: Sun, 11 Dec 2022 02:55:56 -0800
Subject: [PATCH 318/459] Parse Python 3.9+ parenthesized context managers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Since the upstream grammar for this is not LR(1), we abuse LALRPOP
macros and the Into/TryInto traits to build a cover grammar that
converts to either tuples or `with` items after additional validation.
It’s annoying and ugly, but something like this is basically our only
option short of switching to a more powerful parser algorithm.

Fixes #4145.

Signed-off-by: Anders Kaseorg <andersk@mit.edu>
---
 parser/python.lalrpop                         |  216 +-
 parser/src/lib.rs                             |    1 +
 ...n_parser__with__tests__with_statement.snap | 2094 +++++++++++++++++
 parser/src/with.rs                            |  188 ++
 4 files changed, 2413 insertions(+), 86 deletions(-)
 create mode 100644 parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
 create mode 100644 parser/src/with.rs

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 565130cb..1b82e7c8 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -10,7 +10,8 @@ use crate::{
     lexer,
     context::set_context,
     string::parse_strings,
-    token::StringKind
+    token::StringKind,
+    with::{ExprOrWithitems, TupleOrWithitems},
 };
 use num_bigint::BigInt;
 
@@ -159,10 +160,6 @@ TestOrStarExprList: ast::Expr = {
     TestList
 };
 
-TestOrStarNamedExprList: ast::Expr = {
-    GenericList<TestOrStarNamedExpr>
-};
-
 TestOrStarExpr: ast::Expr = {
     Test,
     StarExpr,
@@ -173,6 +170,12 @@ TestOrStarNamedExpr: ast::Expr = {
     StarExpr,
 };
 
+TestOrStarNamedExprOrWithitem: (ast::Expr, Option<Box<ast::Expr>>) = {
+    <e:NamedExpressionTest> => (e, None),
+    <e:StarExpr> => (e, None),
+    <e:Test> "as" <v:Expression> => (e, Some(Box::new(v))),
+}
+
 AugAssign: ast::Operator = {
     "+=" => ast::Operator::Add,
     "-=" => ast::Operator::Sub,
@@ -472,7 +475,7 @@ ExceptClause: ast::Excepthandler = {
 };
 
 WithStatement: ast::Stmt = {
-    <location:@L> <is_async:"async"?> "with" <items:OneOrMore<WithItem>> ":" <body:Suite> <end_location:@R> => {
+    <location:@L> <is_async:"async"?> "with" <items:WithItems> ":" <body:Suite> <end_location:@R> => {
         let type_comment = None;
         let node = if is_async.is_some() {
             ast::StmtKind::AsyncWith { items, body, type_comment }
@@ -483,6 +486,25 @@ WithStatement: ast::Stmt = {
     },
 };
 
+// These are only used for their types as macro parameters
+ExprGoal: ast::Expr = {};
+ExprOrWithitemsGoal: ExprOrWithitems = {};
+
+WithItems: Vec<ast::Withitem> = {
+    <items:TestAs<ExprOrWithitemsGoal>> =>? items.try_into(),
+    <first:TestAs<ExprOrWithitemsGoal>> "as" <vars:Expression> =>? {
+        let optional_vars = Some(Box::new(set_context(vars, ast::ExprContext::Store)));
+        let context_expr = Box::new(first.try_into()?);
+        Ok(vec![ast::Withitem { context_expr, optional_vars }])
+    },
+    <first:TestAs<ExprOrWithitemsGoal>> <n:("as" Expression)?> "," <mut items:OneOrMore<WithItem>> =>? {
+        let optional_vars = n.map(|val| Box::new(set_context(val.1, ast::ExprContext::Store)));
+        let context_expr = Box::new(first.try_into()?);
+        items.insert(0, ast::Withitem { context_expr, optional_vars });
+        Ok(items)
+    }
+};
+
 WithItem: ast::Withitem = {
     <context_expr:Test> <n:("as" Expression)?> => {
         let optional_vars = n.map(|val| Box::new(set_context(val.1, ast::ExprContext::Store)));
@@ -688,7 +710,8 @@ YieldExpr: ast::Expr = {
     },
 };
 
-Test: ast::Expr = {
+Test = TestAs<ExprGoal>;
+TestAs<Goal>: Goal = {
     <body:OrTest> <location:@L> "if" <test:OrTest> "else" <orelse:Test> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
@@ -698,9 +721,9 @@ Test: ast::Expr = {
             body: Box::new(body),
             orelse: Box::new(orelse),
         }
-    },
-    OrTest,
-    LambdaDef,
+    }.into(),
+    OrTestAs<Goal>,
+    <e:LambdaDef> => e.into(),
 };
 
 NamedExpressionTest: ast::Expr = {
@@ -750,7 +773,8 @@ LambdaDef: ast::Expr = {
     }
 }
 
-OrTest: ast::Expr = {
+OrTest = OrTestAs<ExprGoal>;
+OrTestAs<Goal>: Goal = {
     <location:@L> <e1:AndTest> <e2:("or" AndTest)+> <end_location:@R> => {
         let mut values = vec![e1];
         values.extend(e2.into_iter().map(|e| e.1));
@@ -759,12 +783,13 @@ OrTest: ast::Expr = {
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values }
-        }
+        }.into()
     },
-    AndTest,
+    AndTestAs<Goal>,
 };
 
-AndTest: ast::Expr = {
+AndTest = AndTestAs<ExprGoal>;
+AndTestAs<Goal>: Goal = {
     <location:@L> <e1:NotTest> <e2:("and" NotTest)+> <end_location:@R> => {
         let mut values = vec![e1];
         values.extend(e2.into_iter().map(|e| e.1));
@@ -773,22 +798,24 @@ AndTest: ast::Expr = {
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values }
-        }
+        }.into()
     },
-    NotTest,
+    NotTestAs<Goal>,
 };
 
-NotTest: ast::Expr = {
+NotTest = NotTestAs<ExprGoal>;
+NotTestAs<Goal>: Goal = {
     <location:@L> "not" <e:NotTest> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::UnaryOp { operand: Box::new(e), op: ast::Unaryop::Not }
-    },
-    Comparison,
+    }.into(),
+    ComparisonAs<Goal>,
 };
 
-Comparison: ast::Expr = {
+Comparison = ComparisonAs<ExprGoal>;
+ComparisonAs<Goal>: Goal = {
     <location:@L> <left:Expression> <comparisons:(CompOp Expression)+> <end_location:@R> => {
         let (ops, comparators) = comparisons.into_iter().unzip();
         ast::Expr {
@@ -796,9 +823,9 @@ Comparison: ast::Expr = {
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::Compare { left: Box::new(left), ops, comparators }
-        }
+        }.into()
     },
-    Expression,
+    ExpressionAs<Goal>,
 };
 
 CompOp: ast::Cmpop = {
@@ -814,44 +841,48 @@ CompOp: ast::Cmpop = {
     "is" "not" => ast::Cmpop::IsNot,
 };
 
-Expression: ast::Expr = {
+Expression = ExpressionAs<ExprGoal>;
+ExpressionAs<Goal>: Goal = {
     <e1:Expression> <location:@L> "|" <e2:XorExpression> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) }
-    },
-    XorExpression,
+    }.into(),
+    XorExpressionAs<Goal>,
 };
 
-XorExpression: ast::Expr = {
+XorExpression = XorExpressionAs<ExprGoal>;
+XorExpressionAs<Goal>: Goal = {
     <e1:XorExpression> <location:@L> "^" <e2:AndExpression> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) }
-    },
-    AndExpression,
+    }.into(),
+    AndExpressionAs<Goal>,
 };
 
-AndExpression: ast::Expr = {
+AndExpression = AndExpressionAs<ExprGoal>;
+AndExpressionAs<Goal>: Goal = {
     <e1:AndExpression> <location:@L> "&" <e2:ShiftExpression> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) }
-    },
-    ShiftExpression,
+    }.into(),
+    ShiftExpressionAs<Goal>,
 };
 
-ShiftExpression: ast::Expr = {
+ShiftExpression = ShiftExpressionAs<ExprGoal>;
+ShiftExpressionAs<Goal>: Goal = {
     <e1:ShiftExpression> <location:@L> <op:ShiftOp> <e2:ArithmeticExpression> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) }
-    },
-    ArithmeticExpression,
+    }.into(),
+    ArithmeticExpressionAs<Goal>,
 };
 
 ShiftOp: ast::Operator = {
@@ -859,14 +890,15 @@ ShiftOp: ast::Operator = {
     ">>" => ast::Operator::RShift,
 };
 
-ArithmeticExpression: ast::Expr = {
+ArithmeticExpression = ArithmeticExpressionAs<ExprGoal>;
+ArithmeticExpressionAs<Goal>: Goal = {
     <location:@L> <a:ArithmeticExpression> <op:AddOp> <b:Term> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
-    },
-    Term,
+    }.into(),
+    TermAs<Goal>,
 };
 
 AddOp: ast::Operator = {
@@ -874,14 +906,15 @@ AddOp: ast::Operator = {
     "-" => ast::Operator::Sub,
 };
 
-Term: ast::Expr = {
+Term = TermAs<ExprGoal>;
+TermAs<Goal>: Goal = {
     <a:Term> <location:@L> <op:MulOp> <b:Factor> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
-    },
-    Factor,
+    }.into(),
+    FactorAs<Goal>,
 };
 
 MulOp: ast::Operator = {
@@ -892,14 +925,15 @@ MulOp: ast::Operator = {
     "@" => ast::Operator::MatMult,
 };
 
-Factor: ast::Expr = {
+Factor = FactorAs<ExprGoal>;
+FactorAs<Goal>: Goal = {
     <location:@L> <op:UnaryOp> <e:Factor> <end_location:@R>  => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::UnaryOp { operand: Box::new(e), op }
-    },
-    Power,
+    }.into(),
+    PowerAs<Goal>,
 };
 
 UnaryOp: ast::Unaryop = {
@@ -908,48 +942,53 @@ UnaryOp: ast::Unaryop = {
     "~" => ast::Unaryop::Invert,
 };
 
-Power: ast::Expr = {
+Power = PowerAs<ExprGoal>;
+PowerAs<Goal>: Goal = {
     <e:AtomExpr> <location:@L> "**" <b:Factor> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) }
-    },
-    AtomExpr,
+    }.into(),
+    AtomExprAs<Goal>,
 };
 
-AtomExpr: ast::Expr = {
-    <location:@L> "await" <atom:AtomExpr2> <end_location:@R> => ast::Expr {
-        location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Await { value: Box::new(atom) }
+AtomExpr = AtomExprAs<ExprGoal>;
+AtomExprAs<Goal>: Goal = {
+    <location:@L> "await" <atom:AtomExpr2> <end_location:@R> => {
+        ast::Expr {
+            location,
+            end_location: Some(end_location),
+            custom: (),
+            node: ast::ExprKind::Await { value: Box::new(atom) }
+        }.into()
     },
-    AtomExpr2,
+    AtomExpr2As<Goal>,
 }
 
-AtomExpr2: ast::Expr = {
-    Atom,
+AtomExpr2 = AtomExpr2As<ExprGoal>;
+AtomExpr2As<Goal>: Goal = {
+    AtomAs<Goal>,
     <location:@L> <f:AtomExpr2> "(" <a:ArgumentList> ")" <end_location:@R> => {
         ast::Expr {
             location,
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::Call { func: Box::new(f), args: a.args, keywords: a.keywords }
-        }
+        }.into()
     },
     <location:@L> <e:AtomExpr2> "[" <s:SubscriptList> "]" <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Subscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load }
-    },
+    }.into(),
     <location:@L> <e:AtomExpr2> "." <attr:Identifier> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Attribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load }
-    },
+    }.into(),
 };
 
 SubscriptList: ast::Expr = {
@@ -991,20 +1030,21 @@ SliceOp: Option<ast::Expr> = {
     <location:@L> ":" <e:Test?> => e,
 }
 
-Atom: ast::Expr = {
-    <location:@L> <s:(@L string @R)+> =>? parse_strings(s).map_err(|e| e.into()),
+Atom = AtomAs<ExprGoal>;
+AtomAs<Goal>: Goal = {
+    <location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?.into()),
     <location:@L> <value:Constant> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Constant { value, kind: None }
-    },
+    }.into(),
     <location:@L> <name:Identifier> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load }
-    },
+    }.into(),
     <location:@L> "[" <e:ListLiteralValues?> "]"<end_location:@R>  => {
         let elts = e.unwrap_or_default();
         ast::Expr {
@@ -1012,7 +1052,7 @@ Atom: ast::Expr = {
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::List { elts, ctx: ast::ExprContext::Load }
-        }
+        }.into()
     },
     <location:@L> "[" <elt:TestOrStarNamedExpr> <generators:CompFor> "]" <end_location:@R> => {
         ast::Expr {
@@ -1020,34 +1060,38 @@ Atom: ast::Expr = {
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::ListComp { elt: Box::new(elt), generators }
-        }
-    },
-    "(" <elt:TestOrStarNamedExprList> ")" =>? {
-        match elt.node {
-            ast::ExprKind::Starred { .. } => {
-                Err(LexicalError{
-                    error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
-                    location: elt.location,
-                }.into())
-            }
-            _ => {
-                Ok(elt)
+        }.into()
+    },
+    "(" <location:@L> <items:OneOrMore<TestOrStarNamedExprOrWithitem>> <trailing_comma:","?> <end_location:@R> ")" =>? {
+        if items.len() == 1 && items[0].1.is_none() && trailing_comma.is_none() {
+            match items[0].0.node {
+                ast::ExprKind::Starred { .. } => {
+                    Err(LexicalError{
+                        error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
+                        location: items[0].0.location,
+                    }.into())
+                }
+                _ => {
+                    Ok(items.into_iter().next().unwrap().0.into())
+                }
             }
+        } else {
+            TupleOrWithitems { location, end_location, items }.try_into()
         }
     },
     <location:@L> "(" ")" <end_location:@R> => ast::Expr::new(
         location,
         end_location,
         ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load }
-    ),
-    "(" <e:YieldExpr> ")" => e,
+    ).into(),
+    "(" <e:YieldExpr> ")" => e.into(),
     <location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" <end_location:@R> => {
         ast::Expr {
             location,
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::GeneratorExp { elt: Box::new(elt), generators }
-        }
+        }.into()
     },
     "(" <location:@L> "**" <e:Expression> ")" <end_location:@R> =>? {
         Err(LexicalError{
@@ -1099,7 +1143,7 @@ Atom: ast::Expr = {
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::Dict { keys, values }
-        }
+        }.into()
     },
     <location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" <end_location:@R> => {
         ast::Expr {
@@ -1111,26 +1155,26 @@ Atom: ast::Expr = {
                 value: Box::new(e1.1),
                 generators,
             }
-        }
+        }.into()
     },
     <location:@L> "{" <elts:SetLiteralValues> "}" <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Set { elts }
-    },
+    }.into(),
     <location:@L> "{" <elt:Test> <generators:CompFor> "}" <end_location:@R> => {
         ast::Expr {
             location,
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::SetComp { elt: Box::new(elt), generators }
-        }
+        }.into()
     },
-    <location:@L> "True" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: true.into(), kind: None }),
-    <location:@L> "False" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: false.into(), kind: None }),
-    <location:@L> "None" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }),
-    <location:@L> "..." <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }),
+    <location:@L> "True" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: true.into(), kind: None }).into(),
+    <location:@L> "False" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: false.into(), kind: None }).into(),
+    <location:@L> "None" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }).into(),
+    <location:@L> "..." <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }).into(),
 };
 
 ListLiteralValues: Vec<ast::Expr> = {
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index 0bd363a8..66e56537 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -33,3 +33,4 @@ mod python;
 mod context;
 mod string;
 pub mod token;
+mod with;
diff --git a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
new file mode 100644
index 00000000..e82ee2d0
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
@@ -0,0 +1,2094 @@
+---
+source: compiler/parser/src/with.rs
+expression: "parse_program(source, \"<test>\").unwrap()"
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 2,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 1,
+                            column: 5,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 6,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                0,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: None,
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 1,
+                        column: 8,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 12,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 2,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 3,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 2,
+                            column: 5,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 2,
+                                column: 6,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                0,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 2,
+                                column: 10,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 11,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 2,
+                        column: 13,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 2,
+                            column: 17,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 3,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 4,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 3,
+                            column: 5,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 3,
+                                column: 6,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                0,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: None,
+                },
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 3,
+                            column: 8,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 3,
+                                column: 9,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                1,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: None,
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 3,
+                        column: 11,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 3,
+                            column: 15,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 4,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 5,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 4,
+                            column: 5,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 4,
+                                column: 6,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                0,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 4,
+                                column: 10,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 4,
+                                    column: 11,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 4,
+                            column: 13,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 4,
+                                column: 14,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                1,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 4,
+                                column: 18,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 4,
+                                    column: 19,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "y",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 4,
+                        column: 21,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 4,
+                            column: 25,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 5,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 6,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 5,
+                            column: 7,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 5,
+                                column: 18,
+                            },
+                        ),
+                        custom: (),
+                        node: IfExp {
+                            test: Located {
+                                location: Location {
+                                    row: 5,
+                                    column: 10,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 5,
+                                        column: 11,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        1,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                            body: Located {
+                                location: Location {
+                                    row: 5,
+                                    column: 5,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 5,
+                                        column: 6,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        0,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                            orelse: Located {
+                                location: Location {
+                                    row: 5,
+                                    column: 17,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 5,
+                                        column: 18,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        2,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    optional_vars: None,
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 5,
+                        column: 20,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 5,
+                            column: 24,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 6,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 7,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 6,
+                            column: 7,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 6,
+                                column: 18,
+                            },
+                        ),
+                        custom: (),
+                        node: IfExp {
+                            test: Located {
+                                location: Location {
+                                    row: 6,
+                                    column: 10,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 6,
+                                        column: 11,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        1,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                            body: Located {
+                                location: Location {
+                                    row: 6,
+                                    column: 5,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 6,
+                                        column: 6,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        0,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                            orelse: Located {
+                                location: Location {
+                                    row: 6,
+                                    column: 17,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 6,
+                                        column: 18,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        2,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 6,
+                                column: 22,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 6,
+                                    column: 23,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 6,
+                        column: 25,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 6,
+                            column: 29,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 7,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 8,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 7,
+                            column: 5,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 7,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Tuple {
+                            elts: [],
+                            ctx: Load,
+                        },
+                    },
+                    optional_vars: None,
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 7,
+                        column: 9,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 7,
+                            column: 13,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 8,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 9,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 8,
+                            column: 5,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 8,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Tuple {
+                            elts: [],
+                            ctx: Load,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 8,
+                                column: 11,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 8,
+                                    column: 12,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 8,
+                        column: 14,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 8,
+                            column: 18,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 9,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 10,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 9,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 9,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                0,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: None,
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 9,
+                        column: 10,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 9,
+                            column: 14,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 10,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 11,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 10,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 10,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                0,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 10,
+                                column: 12,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 10,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 10,
+                        column: 15,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 10,
+                            column: 19,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 11,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 12,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 11,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 11,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                0,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: None,
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 11,
+                        column: 11,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 11,
+                            column: 15,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 12,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 13,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 12,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 12,
+                                column: 8,
+                            },
+                        ),
+                        custom: (),
+                        node: Tuple {
+                            elts: [
+                                Located {
+                                    location: Location {
+                                        row: 12,
+                                        column: 6,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 12,
+                                            column: 7,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                            ctx: Load,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 12,
+                                column: 13,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 12,
+                                    column: 14,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 12,
+                        column: 16,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 12,
+                            column: 20,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 13,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 14,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 13,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 13,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                0,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: None,
+                },
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 13,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 13,
+                                column: 10,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                1,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: None,
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 13,
+                        column: 13,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 13,
+                            column: 17,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 14,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 15,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 14,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 14,
+                                column: 10,
+                            },
+                        ),
+                        custom: (),
+                        node: Tuple {
+                            elts: [
+                                Located {
+                                    location: Location {
+                                        row: 14,
+                                        column: 6,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 14,
+                                            column: 7,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 14,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 14,
+                                            column: 10,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            1,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                            ctx: Load,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 14,
+                                column: 15,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 14,
+                                    column: 16,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 14,
+                        column: 18,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 14,
+                            column: 22,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 15,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 16,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 15,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 15,
+                                column: 9,
+                            },
+                        ),
+                        custom: (),
+                        node: Tuple {
+                            elts: [
+                                Located {
+                                    location: Location {
+                                        row: 15,
+                                        column: 6,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 15,
+                                            column: 8,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Starred {
+                                        value: Located {
+                                            location: Location {
+                                                row: 15,
+                                                column: 7,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 15,
+                                                    column: 8,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "a",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        ctx: Load,
+                                    },
+                                },
+                            ],
+                            ctx: Load,
+                        },
+                    },
+                    optional_vars: None,
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 15,
+                        column: 12,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 15,
+                            column: 16,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 16,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 17,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 16,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 16,
+                                column: 9,
+                            },
+                        ),
+                        custom: (),
+                        node: Tuple {
+                            elts: [
+                                Located {
+                                    location: Location {
+                                        row: 16,
+                                        column: 6,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 16,
+                                            column: 8,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Starred {
+                                        value: Located {
+                                            location: Location {
+                                                row: 16,
+                                                column: 7,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 16,
+                                                    column: 8,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "a",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        ctx: Load,
+                                    },
+                                },
+                            ],
+                            ctx: Load,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 16,
+                                column: 14,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 16,
+                                    column: 15,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 16,
+                        column: 17,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 16,
+                            column: 21,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 17,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 18,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 17,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 17,
+                                column: 11,
+                            },
+                        ),
+                        custom: (),
+                        node: Tuple {
+                            elts: [
+                                Located {
+                                    location: Location {
+                                        row: 17,
+                                        column: 6,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 17,
+                                            column: 7,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 17,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 17,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Starred {
+                                        value: Located {
+                                            location: Location {
+                                                row: 17,
+                                                column: 10,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 17,
+                                                    column: 11,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "a",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        ctx: Load,
+                                    },
+                                },
+                            ],
+                            ctx: Load,
+                        },
+                    },
+                    optional_vars: None,
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 17,
+                        column: 14,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 17,
+                            column: 18,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 18,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 19,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 18,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 18,
+                                column: 11,
+                            },
+                        ),
+                        custom: (),
+                        node: Tuple {
+                            elts: [
+                                Located {
+                                    location: Location {
+                                        row: 18,
+                                        column: 6,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 18,
+                                            column: 7,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 18,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 18,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Starred {
+                                        value: Located {
+                                            location: Location {
+                                                row: 18,
+                                                column: 10,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 18,
+                                                    column: 11,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "a",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        ctx: Load,
+                                    },
+                                },
+                            ],
+                            ctx: Load,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 18,
+                                column: 16,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 18,
+                                    column: 17,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 18,
+                        column: 19,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 18,
+                            column: 23,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 19,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 20,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 19,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 19,
+                                column: 12,
+                            },
+                        ),
+                        custom: (),
+                        node: NamedExpr {
+                            target: Located {
+                                location: Location {
+                                    row: 19,
+                                    column: 6,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 19,
+                                        column: 12,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "a",
+                                    ctx: Store,
+                                },
+                            },
+                            value: Located {
+                                location: Location {
+                                    row: 19,
+                                    column: 11,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 19,
+                                        column: 12,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        0,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    optional_vars: None,
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 19,
+                        column: 15,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 19,
+                            column: 19,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 20,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 21,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 20,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 20,
+                                column: 12,
+                            },
+                        ),
+                        custom: (),
+                        node: NamedExpr {
+                            target: Located {
+                                location: Location {
+                                    row: 20,
+                                    column: 6,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 20,
+                                        column: 12,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "a",
+                                    ctx: Store,
+                                },
+                            },
+                            value: Located {
+                                location: Location {
+                                    row: 20,
+                                    column: 11,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 20,
+                                        column: 12,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        0,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 20,
+                                column: 17,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 20,
+                                    column: 18,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 20,
+                        column: 20,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 20,
+                            column: 24,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 21,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 22,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 21,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 21,
+                                column: 12,
+                            },
+                        ),
+                        custom: (),
+                        node: NamedExpr {
+                            target: Located {
+                                location: Location {
+                                    row: 21,
+                                    column: 6,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 21,
+                                        column: 12,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "a",
+                                    ctx: Store,
+                                },
+                            },
+                            value: Located {
+                                location: Location {
+                                    row: 21,
+                                    column: 11,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 21,
+                                        column: 12,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        0,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    optional_vars: None,
+                },
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 21,
+                            column: 14,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 21,
+                                column: 20,
+                            },
+                        ),
+                        custom: (),
+                        node: NamedExpr {
+                            target: Located {
+                                location: Location {
+                                    row: 21,
+                                    column: 14,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 21,
+                                        column: 20,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "b",
+                                    ctx: Store,
+                                },
+                            },
+                            value: Located {
+                                location: Location {
+                                    row: 21,
+                                    column: 19,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 21,
+                                        column: 20,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        1,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    optional_vars: None,
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 21,
+                        column: 23,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 21,
+                            column: 27,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 22,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 23,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 22,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 22,
+                                column: 20,
+                            },
+                        ),
+                        custom: (),
+                        node: Tuple {
+                            elts: [
+                                Located {
+                                    location: Location {
+                                        row: 22,
+                                        column: 6,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 22,
+                                            column: 12,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: NamedExpr {
+                                        target: Located {
+                                            location: Location {
+                                                row: 22,
+                                                column: 6,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 22,
+                                                    column: 12,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "a",
+                                                ctx: Store,
+                                            },
+                                        },
+                                        value: Located {
+                                            location: Location {
+                                                row: 22,
+                                                column: 11,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 22,
+                                                    column: 12,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 22,
+                                        column: 14,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 22,
+                                            column: 20,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: NamedExpr {
+                                        target: Located {
+                                            location: Location {
+                                                row: 22,
+                                                column: 14,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 22,
+                                                    column: 20,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "b",
+                                                ctx: Store,
+                                            },
+                                        },
+                                        value: Located {
+                                            location: Location {
+                                                row: 22,
+                                                column: 19,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 22,
+                                                    column: 20,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    1,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                            ctx: Load,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 22,
+                                column: 25,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 22,
+                                    column: 26,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 22,
+                        column: 28,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 22,
+                            column: 32,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+]
diff --git a/parser/src/with.rs b/parser/src/with.rs
new file mode 100644
index 00000000..afc48236
--- /dev/null
+++ b/parser/src/with.rs
@@ -0,0 +1,188 @@
+//! Intermediate types for `with` statement cover grammar.
+//!
+//! When we start parsing a `with` statement, we don't initially know
+//! whether we're looking at a tuple or a Python 3.9+ parenthesized
+//! collection of contexts:
+//!
+//! ```python
+//! with (a, b, c) as t:  # tuple
+//! with (a, b, c):  # withitems
+//! ```
+//!
+//! Since LALRPOP requires us to commit to an output type before we
+//! have enough information to decide, we build a cover grammar that's
+//! convertible either way.  This module contains the necessary
+//! intermediate data types.
+
+use crate::ast::{self, Location};
+use crate::error::{LexicalError, LexicalErrorType};
+use crate::token::Tok;
+use lalrpop_util::ParseError as LalrpopError;
+
+/// Represents a parenthesized collection that we might later convert
+/// to a tuple or to `with` items.
+///
+/// It can be converted to either `Expr` or `ExprOrWithitems` with
+/// `.try_into()`.  The `Expr` conversion will fail if any `as`
+/// variables are present.  The `ExprOrWithitems` conversion cannot
+/// fail (but we need it to have the same interface so we can use
+/// LALRPOP macros to declare the cover grammar without much code
+/// duplication).
+pub struct TupleOrWithitems {
+    pub location: Location,
+    pub end_location: Location,
+    pub items: Vec<(ast::Expr, Option<Box<ast::Expr>>)>,
+}
+
+impl TryFrom<TupleOrWithitems> for ast::Expr {
+    type Error = LalrpopError<Location, Tok, LexicalError>;
+    fn try_from(tuple_or_withitems: TupleOrWithitems) -> Result<ast::Expr, Self::Error> {
+        Ok(ast::Expr {
+            location: tuple_or_withitems.location,
+            end_location: Some(tuple_or_withitems.end_location),
+            custom: (),
+            node: ast::ExprKind::Tuple {
+                elts: tuple_or_withitems
+                    .items
+                    .into_iter()
+                    .map(|(expr, optional_vars)| {
+                        if let Some(vars) = optional_vars {
+                            Err(LexicalError {
+                                error: LexicalErrorType::OtherError(
+                                    "cannot use 'as' here".to_string(),
+                                ),
+                                location: vars.location,
+                            })?
+                        }
+                        Ok(expr)
+                    })
+                    .collect::<Result<Vec<ast::Expr>, Self::Error>>()?,
+                ctx: ast::ExprContext::Load,
+            },
+        })
+    }
+}
+
+impl TryFrom<TupleOrWithitems> for ExprOrWithitems {
+    type Error = LalrpopError<Location, Tok, LexicalError>;
+    fn try_from(items: TupleOrWithitems) -> Result<ExprOrWithitems, Self::Error> {
+        Ok(ExprOrWithitems::TupleOrWithitems(items))
+    }
+}
+
+/// Represents either a non-tuple expression, or a parenthesized
+/// collection that we might later convert to a tuple or to `with`
+/// items.
+///
+/// It can be constructed from an `Expr` with `.into()`.  (The same
+/// interface can be used to convert an `Expr` into itself, which is
+/// also important for our LALRPOP macro setup.)
+///
+/// It can be converted to either `Expr` or `Vec<Withitem>` with
+/// `.try_into()`.  The `Expr` conversion will fail if any `as`
+/// clauses are present.  The `Vec<Withitem>` conversion will fail if
+/// both `as` clauses and starred expressions are present.
+pub enum ExprOrWithitems {
+    Expr(ast::Expr),
+    TupleOrWithitems(TupleOrWithitems),
+}
+
+impl From<ast::Expr> for ExprOrWithitems {
+    fn from(expr: ast::Expr) -> ExprOrWithitems {
+        ExprOrWithitems::Expr(expr)
+    }
+}
+
+impl TryFrom<ExprOrWithitems> for ast::Expr {
+    type Error = LalrpopError<Location, Tok, LexicalError>;
+    fn try_from(expr_or_withitems: ExprOrWithitems) -> Result<ast::Expr, Self::Error> {
+        match expr_or_withitems {
+            ExprOrWithitems::Expr(expr) => Ok(expr),
+            ExprOrWithitems::TupleOrWithitems(items) => items.try_into(),
+        }
+    }
+}
+
+impl TryFrom<ExprOrWithitems> for Vec<ast::Withitem> {
+    type Error = LalrpopError<Location, Tok, LexicalError>;
+    fn try_from(expr_or_withitems: ExprOrWithitems) -> Result<Vec<ast::Withitem>, Self::Error> {
+        match expr_or_withitems {
+            ExprOrWithitems::TupleOrWithitems(tuple_or_withitems)
+                if !tuple_or_withitems.items.iter().any(|(context_expr, _)| {
+                    matches!(context_expr.node, ast::ExprKind::Starred { .. })
+                }) =>
+            {
+                Ok(tuple_or_withitems
+                    .items
+                    .into_iter()
+                    .map(|(context_expr, optional_vars)| ast::Withitem {
+                        context_expr: Box::new(context_expr),
+                        optional_vars,
+                    })
+                    .collect())
+            }
+            _ => Ok(vec![ast::Withitem {
+                context_expr: Box::new(expr_or_withitems.try_into()?),
+                optional_vars: None,
+            }]),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::parser::parse_program;
+
+    #[test]
+    fn test_with_statement() {
+        let source = "\
+with 0: pass
+with 0 as x: pass
+with 0, 1: pass
+with 0 as x, 1 as y: pass
+with 0 if 1 else 2: pass
+with 0 if 1 else 2 as x: pass
+with (): pass
+with () as x: pass
+with (0): pass
+with (0) as x: pass
+with (0,): pass
+with (0,) as x: pass
+with (0, 1): pass
+with (0, 1) as x: pass
+with (*a,): pass
+with (*a,) as x: pass
+with (0, *a): pass
+with (0, *a) as x: pass
+with (a := 0): pass
+with (a := 0) as x: pass
+with (a := 0, b := 1): pass
+with (a := 0, b := 1) as x: pass
+";
+        insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
+    }
+
+    #[test]
+    fn test_with_statement_invalid() {
+        for source in [
+            "with 0,: pass",
+            "with 0 as x,: pass",
+            "with 0 as *x: pass",
+            "with *a: pass",
+            "with *a as x: pass",
+            "with (*a): pass",
+            "with (*a) as x: pass",
+            "with *a, 0 as x: pass",
+            "with (*a, 0 as x): pass",
+            "with 0 as x, *a: pass",
+            "with (0 as x, *a): pass",
+            "with (0 as x) as y: pass",
+            "with (0 as x), 1: pass",
+            "with ((0 as x)): pass",
+            "with a := 0 as x: pass",
+            "with (a := 0 as x): pass",
+        ] {
+            assert!(parse_program(source, "<test>").is_err());
+        }
+    }
+}

From 3d8a2458705aee79cfbcd720a9f2a6065f766989 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Mon, 12 Dec 2022 22:14:05 +0900
Subject: [PATCH 319/459] Address comments

Signed-off-by: harupy <hkawamura0130@gmail.com>
---
 parser/python.lalrpop | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index ff33fe4f..3b78efc3 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -361,10 +361,11 @@ IfStatement: ast::Stmt = {
         // Determine last else:
         let mut last = s3.map(|s| s.2).unwrap_or_default();
         let end_location = last
-            .last()
-            .map(|last| last.end_location)
-            .or_else(|| s2.last().map(|last| last.4.last().unwrap().end_location))
-            .unwrap_or_else(|| body.last().unwrap().end_location);
+	            .last()
+	            .or_else(|| s2.last().and_then(|last| last.4.last()))
+	            .or_else(|| body.last())
+	            .unwrap()
+	            .end_location;
         // handle elif:
         for i in s2.into_iter().rev() {
             let x = ast::Stmt {
@@ -387,11 +388,12 @@ IfStatement: ast::Stmt = {
 
 WhileStatement: ast::Stmt = {
     <location:@L> "while" <test:NamedExpressionTest> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
-        let end_location = s2
-            .as_ref()
-            .map(|s| s.2.last().unwrap().end_location)
-            .unwrap_or_else(|| body.last().unwrap().end_location);
         let orelse = s2.map(|s| s.2).unwrap_or_default();
+        let end_location = orelse
+            .last()
+            .or_else(|| body.last())
+            .unwrap()
+            .end_location;
         ast::Stmt {
             custom: (),
             location,
@@ -407,12 +409,13 @@ WhileStatement: ast::Stmt = {
 
 ForStatement: ast::Stmt = {
     <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:TestList> ":" <body:Suite> <s2:("else" ":" Suite)?> => {
-        let end_location = s2
-            .as_ref()
-            .map(|s| s.2.last().unwrap().end_location)
-            .unwrap_or_else(|| body.last().unwrap().end_location)
-            .unwrap();
         let orelse = s2.map(|s| s.2).unwrap_or_default();
+        let end_location = orelse
+            .last()
+            .or_else(|| body.last())
+            .unwrap()
+            .end_location
+            .unwrap();
         let target = Box::new(set_context(target, ast::ExprContext::Store));
         let iter = Box::new(iter);
         let type_comment = None;

From 16f2e826e438c01d4ed835bcc0a7e735e3448644 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Mon, 12 Dec 2022 22:18:26 +0900
Subject: [PATCH 320/459] Format

---
 parser/python.lalrpop | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 32bf8343..3c980b06 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -364,11 +364,11 @@ IfStatement: ast::Stmt = {
         // Determine last else:
         let mut last = s3.map(|s| s.2).unwrap_or_default();
         let end_location = last
-	            .last()
-	            .or_else(|| s2.last().and_then(|last| last.4.last()))
-	            .or_else(|| body.last())
-	            .unwrap()
-	            .end_location;
+            .last()
+            .or_else(|| s2.last().and_then(|last| last.4.last()))
+            .or_else(|| body.last())
+            .unwrap()
+            .end_location;
         // handle elif:
         for i in s2.into_iter().rev() {
             let x = ast::Stmt {

From f5ce9517fd09595966b776e5c682ea30a40363fc Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Mon, 12 Dec 2022 22:26:03 +0900
Subject: [PATCH 321/459] Update snapshot

---
 ...n_parser__with__tests__with_statement.snap | 88 +++++++++----------
 1 file changed, 44 insertions(+), 44 deletions(-)

diff --git a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
index e82ee2d0..a36af82d 100644
--- a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
+++ b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
@@ -10,8 +10,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 2,
-                column: 0,
+                row: 1,
+                column: 12,
             },
         ),
         custom: (),
@@ -66,8 +66,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 3,
-                column: 0,
+                row: 2,
+                column: 17,
             },
         ),
         custom: (),
@@ -140,8 +140,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 4,
-                column: 0,
+                row: 3,
+                column: 15,
             },
         ),
         custom: (),
@@ -218,8 +218,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 5,
-                column: 0,
+                row: 4,
+                column: 25,
             },
         ),
         custom: (),
@@ -332,8 +332,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 6,
-                column: 0,
+                row: 5,
+                column: 24,
             },
         ),
         custom: (),
@@ -441,8 +441,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 7,
-                column: 0,
+                row: 6,
+                column: 29,
             },
         ),
         custom: (),
@@ -568,8 +568,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 8,
-                column: 0,
+                row: 7,
+                column: 13,
             },
         ),
         custom: (),
@@ -622,8 +622,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 9,
-                column: 0,
+                row: 8,
+                column: 18,
             },
         ),
         custom: (),
@@ -694,8 +694,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 10,
-                column: 0,
+                row: 9,
+                column: 14,
             },
         ),
         custom: (),
@@ -750,8 +750,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 11,
-                column: 0,
+                row: 10,
+                column: 19,
             },
         ),
         custom: (),
@@ -824,8 +824,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 12,
-                column: 0,
+                row: 11,
+                column: 15,
             },
         ),
         custom: (),
@@ -880,8 +880,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 13,
-                column: 0,
+                row: 12,
+                column: 20,
             },
         ),
         custom: (),
@@ -972,8 +972,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 14,
-                column: 0,
+                row: 13,
+                column: 17,
             },
         ),
         custom: (),
@@ -1050,8 +1050,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 15,
-                column: 0,
+                row: 14,
+                column: 22,
             },
         ),
         custom: (),
@@ -1161,8 +1161,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 16,
-                column: 0,
+                row: 15,
+                column: 16,
             },
         ),
         custom: (),
@@ -1249,8 +1249,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 17,
-                column: 0,
+                row: 16,
+                column: 21,
             },
         ),
         custom: (),
@@ -1355,8 +1355,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 18,
-                column: 0,
+                row: 17,
+                column: 18,
             },
         ),
         custom: (),
@@ -1462,8 +1462,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 19,
-                column: 0,
+                row: 18,
+                column: 23,
             },
         ),
         custom: (),
@@ -1587,8 +1587,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 20,
-                column: 0,
+                row: 19,
+                column: 19,
             },
         ),
         custom: (),
@@ -1675,8 +1675,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 21,
-                column: 0,
+                row: 20,
+                column: 24,
             },
         ),
         custom: (),
@@ -1781,8 +1781,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 22,
-                column: 0,
+                row: 21,
+                column: 27,
             },
         ),
         custom: (),
@@ -1923,8 +1923,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 23,
-                column: 0,
+                row: 22,
+                column: 32,
             },
         ),
         custom: (),

From d13b27dcf05606cf96a96d7f05825a84716ee8b8 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Mon, 12 Dec 2022 22:36:34 +0900
Subject: [PATCH 322/459] Refactor

---
 parser/python.lalrpop | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 3c980b06..cca2a46e 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -439,7 +439,8 @@ TryStatement: ast::Stmt = {
             .last()
             .map(|last| last.end_location)
             .or_else(|| orelse.last().map(|last| last.end_location))
-            .unwrap_or_else(|| handlers.last().unwrap().end_location);
+            .or_else(|| handlers.last().map(|last| last.end_location))
+            .unwrap();
         ast::Stmt {
             custom: (),
             location,

From 36f5f5affb1c172e64fb8d13c4bb10af7825d7ee Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Mon, 12 Dec 2022 09:01:17 -0500
Subject: [PATCH 323/459] Set ExprContext::Store on parenthesized with
 expressions

---
 ...n_parser__with__tests__with_statement.snap | 376 ++++++++++++++++++
 parser/src/with.rs                            |   9 +-
 2 files changed, 384 insertions(+), 1 deletion(-)

diff --git a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
index e82ee2d0..39001dcb 100644
--- a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
+++ b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
@@ -2091,4 +2091,380 @@ expression: "parse_program(source, \"<test>\").unwrap()"
             type_comment: None,
         },
     },
+    Located {
+        location: Location {
+            row: 23,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 24,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 23,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 23,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                0,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 23,
+                                column: 11,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 23,
+                                    column: 12,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "a",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 23,
+                        column: 15,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 23,
+                            column: 19,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 24,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 25,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 24,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 24,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                0,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 24,
+                                column: 11,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 24,
+                                    column: 12,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "a",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 24,
+                        column: 16,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 24,
+                            column: 20,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 25,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 26,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 25,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 25,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                0,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 25,
+                                column: 11,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 25,
+                                    column: 12,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "a",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 25,
+                            column: 14,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 25,
+                                column: 15,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                1,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 25,
+                                column: 19,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 25,
+                                    column: 20,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "b",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 25,
+                        column: 23,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 25,
+                            column: 27,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 26,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 27,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: With {
+            items: [
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 26,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 26,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                0,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 26,
+                                column: 11,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 26,
+                                    column: 12,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "a",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+                Withitem {
+                    context_expr: Located {
+                        location: Location {
+                            row: 26,
+                            column: 14,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 26,
+                                column: 15,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                1,
+                            ),
+                            kind: None,
+                        },
+                    },
+                    optional_vars: Some(
+                        Located {
+                            location: Location {
+                                row: 26,
+                                column: 19,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 26,
+                                    column: 20,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "b",
+                                ctx: Store,
+                            },
+                        },
+                    ),
+                },
+            ],
+            body: [
+                Located {
+                    location: Location {
+                        row: 26,
+                        column: 24,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 26,
+                            column: 28,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            type_comment: None,
+        },
+    },
 ]
diff --git a/parser/src/with.rs b/parser/src/with.rs
index afc48236..57ee1a6c 100644
--- a/parser/src/with.rs
+++ b/parser/src/with.rs
@@ -15,6 +15,7 @@
 //! intermediate data types.
 
 use crate::ast::{self, Location};
+use crate::context;
 use crate::error::{LexicalError, LexicalErrorType};
 use crate::token::Tok;
 use lalrpop_util::ParseError as LalrpopError;
@@ -117,7 +118,9 @@ impl TryFrom<ExprOrWithitems> for Vec<ast::Withitem> {
                     .into_iter()
                     .map(|(context_expr, optional_vars)| ast::Withitem {
                         context_expr: Box::new(context_expr),
-                        optional_vars,
+                        optional_vars: optional_vars.map(|expr| {
+                            Box::new(context::set_context(*expr, ast::ExprContext::Store))
+                        }),
                     })
                     .collect())
             }
@@ -158,6 +161,10 @@ with (a := 0): pass
 with (a := 0) as x: pass
 with (a := 0, b := 1): pass
 with (a := 0, b := 1) as x: pass
+with (0 as a): pass
+with (0 as a,): pass
+with (0 as a, 1 as b): pass
+with (0 as a, 1 as b,): pass
 ";
         insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
     }

From f126b79b938fcfafb878d0c6f3461eddff3b5a9c Mon Sep 17 00:00:00 2001
From: Dimitris Fasarakis Hilliard <d.f.hilliard@gmail.com>
Date: Tue, 13 Dec 2022 13:28:17 +0200
Subject: [PATCH 324/459] Fix end location in with statements.

---
 ...thon_parser__with__tests__with_statement.snap | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
index b4880f37..ce0c20ed 100644
--- a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
+++ b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
@@ -2098,8 +2098,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 24,
-                column: 0,
+                row: 23,
+                column: 19,
             },
         ),
         custom: (),
@@ -2172,8 +2172,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 25,
-                column: 0,
+                row: 24,
+                column: 20,
             },
         ),
         custom: (),
@@ -2246,8 +2246,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 26,
-                column: 0,
+                row: 25,
+                column: 27,
             },
         ),
         custom: (),
@@ -2360,8 +2360,8 @@ expression: "parse_program(source, \"<test>\").unwrap()"
         },
         end_location: Some(
             Location {
-                row: 27,
-                column: 0,
+                row: 26,
+                column: 28,
             },
         ),
         custom: (),

From 408d15f608aa3cd556d07815466dde06f5c659ee Mon Sep 17 00:00:00 2001
From: Anders Kaseorg <andersk@mit.edu>
Date: Sun, 11 Dec 2022 15:39:47 -0800
Subject: [PATCH 325/459] Remove unnecessary boxing of ASDL product children

Signed-off-by: Anders Kaseorg <andersk@mit.edu>
---
 ast/asdl_rs.py         |  4 +++-
 ast/src/ast_gen.rs     | 10 +++++-----
 parser/python.lalrpop  |  9 ++++-----
 parser/src/function.rs |  5 +----
 parser/src/with.rs     |  4 ++--
 5 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py
index 06ee2e57..ed5800f5 100755
--- a/ast/asdl_rs.py
+++ b/ast/asdl_rs.py
@@ -89,6 +89,7 @@ def __init__(self, name):
         self.has_userdata = None
         self.children = set()
         self.boxed = False
+        self.product = False
 
     def __repr__(self):
         return f"<TypeInfo: {self.name}>"
@@ -145,6 +146,7 @@ def visitProduct(self, product, name):
             info.has_userdata = True
         if len(product.fields) > 2:
             info.boxed = True
+        info.product = True
         self.add_children(name, product.fields)
 
     def add_children(self, name, fields):
@@ -236,7 +238,7 @@ def visitField(self, field, parent, vis, depth):
         if fieldtype and fieldtype.has_userdata:
             typ = f"{typ}<U>"
         # don't box if we're doing Vec<T>, but do box if we're doing Vec<Option<Box<T>>>
-        if fieldtype and fieldtype.boxed and (not field.seq or field.opt):
+        if fieldtype and fieldtype.boxed and (not (parent.product or field.seq) or field.opt):
             typ = f"Box<{typ}>"
         if field.opt:
             typ = f"Option<{typ}>"
diff --git a/ast/src/ast_gen.rs b/ast/src/ast_gen.rs
index 1917bec9..241ebe13 100644
--- a/ast/src/ast_gen.rs
+++ b/ast/src/ast_gen.rs
@@ -337,8 +337,8 @@ pub enum Cmpop {
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct Comprehension<U = ()> {
-    pub target: Box<Expr<U>>,
-    pub iter: Box<Expr<U>>,
+    pub target: Expr<U>,
+    pub iter: Expr<U>,
     pub ifs: Vec<Expr<U>>,
     pub is_async: usize,
 }
@@ -375,7 +375,7 @@ pub type Arg<U = ()> = Located<ArgData<U>, U>;
 #[derive(Clone, Debug, PartialEq)]
 pub struct KeywordData<U = ()> {
     pub arg: Option<Ident>,
-    pub value: Box<Expr<U>>,
+    pub value: Expr<U>,
 }
 pub type Keyword<U = ()> = Located<KeywordData<U>, U>;
 
@@ -388,13 +388,13 @@ pub type Alias<U = ()> = Located<AliasData, U>;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct Withitem<U = ()> {
-    pub context_expr: Box<Expr<U>>,
+    pub context_expr: Expr<U>,
     pub optional_vars: Option<Box<Expr<U>>>,
 }
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct MatchCase<U = ()> {
-    pub pattern: Box<Pattern<U>>,
+    pub pattern: Pattern<U>,
     pub guard: Option<Box<Expr<U>>>,
     pub body: Vec<Stmt<U>>,
 }
diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index cca2a46e..c2cf616b 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -520,12 +520,12 @@ WithItems: Vec<ast::Withitem> = {
     <items:TestAs<ExprOrWithitemsGoal>> =>? items.try_into(),
     <first:TestAs<ExprOrWithitemsGoal>> "as" <vars:Expression> =>? {
         let optional_vars = Some(Box::new(set_context(vars, ast::ExprContext::Store)));
-        let context_expr = Box::new(first.try_into()?);
+        let context_expr = first.try_into()?;
         Ok(vec![ast::Withitem { context_expr, optional_vars }])
     },
     <first:TestAs<ExprOrWithitemsGoal>> <n:("as" Expression)?> "," <mut items:OneOrMore<WithItem>> =>? {
         let optional_vars = n.map(|val| Box::new(set_context(val.1, ast::ExprContext::Store)));
-        let context_expr = Box::new(first.try_into()?);
+        let context_expr = first.try_into()?;
         items.insert(0, ast::Withitem { context_expr, optional_vars });
         Ok(items)
     }
@@ -534,7 +534,6 @@ WithItems: Vec<ast::Withitem> = {
 WithItem: ast::Withitem = {
     <context_expr:Test> <n:("as" Expression)?> => {
         let optional_vars = n.map(|val| Box::new(set_context(val.1, ast::ExprContext::Store)));
-        let context_expr = Box::new(context_expr);
         ast::Withitem { context_expr, optional_vars }
     },
 };
@@ -1280,8 +1279,8 @@ SingleForComprehension: ast::Comprehension = {
     <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest> <ifs:ComprehensionIf*> <end_location:@R> => {
         let is_async = is_async.is_some();
         ast::Comprehension {
-            target: Box::new(set_context(target, ast::ExprContext::Store)),
-            iter: Box::new(iter),
+            target: set_context(target, ast::ExprContext::Store),
+            iter,
             ifs,
             is_async: if is_async { 1 } else { 0 },
         }
diff --git a/parser/src/function.rs b/parser/src/function.rs
index bc4d5142..0cc934fb 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -72,10 +72,7 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, Lexi
                 keywords.push(ast::Keyword::new(
                     start,
                     end,
-                    ast::KeywordData {
-                        arg: name,
-                        value: Box::new(value),
-                    },
+                    ast::KeywordData { arg: name, value },
                 ));
             }
             None => {
diff --git a/parser/src/with.rs b/parser/src/with.rs
index 57ee1a6c..5f018c6b 100644
--- a/parser/src/with.rs
+++ b/parser/src/with.rs
@@ -117,7 +117,7 @@ impl TryFrom<ExprOrWithitems> for Vec<ast::Withitem> {
                     .items
                     .into_iter()
                     .map(|(context_expr, optional_vars)| ast::Withitem {
-                        context_expr: Box::new(context_expr),
+                        context_expr,
                         optional_vars: optional_vars.map(|expr| {
                             Box::new(context::set_context(*expr, ast::ExprContext::Store))
                         }),
@@ -125,7 +125,7 @@ impl TryFrom<ExprOrWithitems> for Vec<ast::Withitem> {
                     .collect())
             }
             _ => Ok(vec![ast::Withitem {
-                context_expr: Box::new(expr_or_withitems.try_into()?),
+                context_expr: expr_or_withitems.try_into()?,
                 optional_vars: None,
             }]),
         }

From 66d79ce568f1a099580600dda566cfb09e459e44 Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Sun, 18 Dec 2022 09:48:35 +0900
Subject: [PATCH 326/459] Fix symbol scope

---
 codegen/src/symboltable.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 5a625b25..3cb2939c 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -483,7 +483,7 @@ impl SymbolTableAnalyzer {
                     };
                 } else {
                     let mut cloned_sym = symbol.clone();
-                    cloned_sym.scope = SymbolScope::Local;
+                    cloned_sym.scope = SymbolScope::Cell;
                     last.0.insert(cloned_sym.name.to_owned(), cloned_sym);
                 }
             }

From af7aac70df01de8fe1ee96d04134863ae92577f2 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 18 Dec 2022 20:53:30 +0900
Subject: [PATCH 327/459] Fix locations of parethesized expressions

---
 parser/python.lalrpop                         |  2 +-
 ...ser__context__tests__assign_attribute.snap |  4 +--
 ...on_parser__context__tests__assign_for.snap |  4 +--
 ...n_parser__context__tests__assign_list.snap |  4 +--
 ...ser__context__tests__assign_list_comp.snap |  4 +--
 ...n_parser__context__tests__assign_name.snap |  4 +--
 ...rser__context__tests__assign_set_comp.snap |  4 +--
 ...arser__context__tests__assign_starred.snap |  8 +++---
 ...ser__context__tests__assign_subscript.snap |  4 +--
 ..._parser__context__tests__assign_tuple.snap |  8 +++---
 ..._context__tests__aug_assign_attribute.snap |  4 +--
 ..._context__tests__aug_assign_subscript.snap |  4 +--
 ...n_parser__with__tests__with_statement.snap | 28 +++++++++----------
 13 files changed, 41 insertions(+), 41 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index cca2a46e..f3fa3e99 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -1090,7 +1090,7 @@ AtomAs<Goal>: Goal = {
             node: ast::ExprKind::ListComp { elt: Box::new(elt), generators }
         }.into()
     },
-    "(" <location:@L> <items:OneOrMore<TestOrStarNamedExprOrWithitem>> <trailing_comma:","?> <end_location:@R> ")" =>? {
+    <location:@L> "(" <items:OneOrMore<TestOrStarNamedExprOrWithitem>> <trailing_comma:","?> ")" <end_location:@R> =>? {
         if items.len() == 1 && items[0].1.is_none() && trailing_comma.is_none() {
             match items[0].0.node {
                 ast::ExprKind::Starred { .. } => {
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap
index a247b566..984cb7a3 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_attribute.snap
@@ -55,12 +55,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 7,
+                    column: 6,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 14,
+                        column: 15,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap
index bbacec76..9dd28020 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_for.snap
@@ -36,12 +36,12 @@ expression: parse_ast
             iter: Located {
                 location: Location {
                     row: 1,
-                    column: 10,
+                    column: 9,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 17,
+                        column: 18,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap
index c90e5a35..ea59293b 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_list.snap
@@ -73,12 +73,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 10,
+                    column: 9,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 17,
+                        column: 18,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap
index 9b6d6ac9..367f09a4 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_list_comp.snap
@@ -87,12 +87,12 @@ expression: parse_ast
                             iter: Located {
                                 location: Location {
                                     row: 1,
-                                    column: 17,
+                                    column: 16,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 24,
+                                        column: 25,
                                     },
                                 ),
                                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap
index e0cdb858..55d4e70d 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_name.snap
@@ -38,12 +38,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 5,
+                    column: 4,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 12,
+                        column: 13,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap
index 84dba820..97ecaa08 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_set_comp.snap
@@ -87,12 +87,12 @@ expression: parse_ast
                             iter: Located {
                                 location: Location {
                                     row: 1,
-                                    column: 17,
+                                    column: 16,
                                 },
                                 end_location: Some(
                                     Location {
                                         row: 1,
-                                        column: 24,
+                                        column: 25,
                                     },
                                 ),
                                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap
index 2e4256e2..015f9bdd 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_starred.snap
@@ -20,12 +20,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 1,
+                        column: 0,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 6,
+                            column: 7,
                         },
                     ),
                     custom: (),
@@ -89,12 +89,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 11,
+                    column: 10,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 18,
+                        column: 19,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap
index 28420ebc..756867c3 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_subscript.snap
@@ -71,12 +71,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 8,
+                    column: 7,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 15,
+                        column: 16,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap
index 30a51df8..c520427d 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_tuple.snap
@@ -20,12 +20,12 @@ expression: parse_ast
                 Located {
                     location: Location {
                         row: 1,
-                        column: 1,
+                        column: 0,
                     },
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 5,
+                            column: 6,
                         },
                     ),
                     custom: (),
@@ -73,12 +73,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 10,
+                    column: 9,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 17,
+                        column: 18,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap
index 3ba9d045..3e6e4b79 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_attribute.snap
@@ -54,12 +54,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 8,
+                    column: 7,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 15,
+                        column: 16,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap
index ebb262a3..72fe61db 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__aug_assign_subscript.snap
@@ -70,12 +70,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 9,
+                    column: 8,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 16,
+                        column: 17,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
index ce0c20ed..031d6d83 100644
--- a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
+++ b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
@@ -891,12 +891,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                     context_expr: Located {
                         location: Location {
                             row: 12,
-                            column: 6,
+                            column: 5,
                         },
                         end_location: Some(
                             Location {
                                 row: 12,
-                                column: 8,
+                                column: 9,
                             },
                         ),
                         custom: (),
@@ -1061,12 +1061,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                     context_expr: Located {
                         location: Location {
                             row: 14,
-                            column: 6,
+                            column: 5,
                         },
                         end_location: Some(
                             Location {
                                 row: 14,
-                                column: 10,
+                                column: 11,
                             },
                         ),
                         custom: (),
@@ -1172,12 +1172,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                     context_expr: Located {
                         location: Location {
                             row: 15,
-                            column: 6,
+                            column: 5,
                         },
                         end_location: Some(
                             Location {
                                 row: 15,
-                                column: 9,
+                                column: 10,
                             },
                         ),
                         custom: (),
@@ -1260,12 +1260,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                     context_expr: Located {
                         location: Location {
                             row: 16,
-                            column: 6,
+                            column: 5,
                         },
                         end_location: Some(
                             Location {
                                 row: 16,
-                                column: 9,
+                                column: 10,
                             },
                         ),
                         custom: (),
@@ -1366,12 +1366,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                     context_expr: Located {
                         location: Location {
                             row: 17,
-                            column: 6,
+                            column: 5,
                         },
                         end_location: Some(
                             Location {
                                 row: 17,
-                                column: 11,
+                                column: 12,
                             },
                         ),
                         custom: (),
@@ -1473,12 +1473,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                     context_expr: Located {
                         location: Location {
                             row: 18,
-                            column: 6,
+                            column: 5,
                         },
                         end_location: Some(
                             Location {
                                 row: 18,
-                                column: 11,
+                                column: 12,
                             },
                         ),
                         custom: (),
@@ -1934,12 +1934,12 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                     context_expr: Located {
                         location: Location {
                             row: 22,
-                            column: 6,
+                            column: 5,
                         },
                         end_location: Some(
                             Location {
                                 row: 22,
-                                column: 20,
+                                column: 21,
                             },
                         ),
                         custom: (),

From a60c7b2fb2ba4ed452cc4bdc49cb3c13eaac7758 Mon Sep 17 00:00:00 2001
From: Noa <coolreader18@gmail.com>
Date: Mon, 19 Dec 2022 18:48:07 -0600
Subject: [PATCH 328/459] Split derive-impl and derive

---
 src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib.rs b/src/lib.rs
index c488a400..1a8d6f32 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,5 +1,4 @@
 use rustpython_codegen::{compile, symboltable};
-use rustpython_compiler_core::CodeObject;
 use rustpython_parser::{
     ast::{fold::Fold, ConstantOptimizer},
     error::ParseErrorType,
@@ -7,6 +6,7 @@ use rustpython_parser::{
 };
 
 pub use rustpython_codegen::compile::CompileOpts;
+pub use rustpython_compiler_core::CodeObject;
 pub use rustpython_compiler_core::{BaseError as CompileErrorBody, Mode};
 
 #[derive(Debug, thiserror::Error)]

From 2c6149707460154b826e1499db0c86a7c532f275 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Wed, 21 Dec 2022 22:05:05 +0900
Subject: [PATCH 329/459] Fix the location of BinOp

---
 parser/python.lalrpop                                | 12 ++++++------
 ...stpython_parser__parser__tests__parse_lambda.snap |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index b1dea045..c1f6ee07 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -870,7 +870,7 @@ CompOp: ast::Cmpop = {
 
 Expression = ExpressionAs<ExprGoal>;
 ExpressionAs<Goal>: Goal = {
-    <e1:Expression> <location:@L> "|" <e2:XorExpression> <end_location:@R> => ast::Expr {
+    <location:@L> <e1:Expression> "|" <e2:XorExpression> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
@@ -881,7 +881,7 @@ ExpressionAs<Goal>: Goal = {
 
 XorExpression = XorExpressionAs<ExprGoal>;
 XorExpressionAs<Goal>: Goal = {
-    <e1:XorExpression> <location:@L> "^" <e2:AndExpression> <end_location:@R> => ast::Expr {
+     <location:@L> <e1:XorExpression> "^" <e2:AndExpression> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
@@ -892,7 +892,7 @@ XorExpressionAs<Goal>: Goal = {
 
 AndExpression = AndExpressionAs<ExprGoal>;
 AndExpressionAs<Goal>: Goal = {
-    <e1:AndExpression> <location:@L> "&" <e2:ShiftExpression> <end_location:@R> => ast::Expr {
+    <location:@L> <e1:AndExpression> "&" <e2:ShiftExpression> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
@@ -903,7 +903,7 @@ AndExpressionAs<Goal>: Goal = {
 
 ShiftExpression = ShiftExpressionAs<ExprGoal>;
 ShiftExpressionAs<Goal>: Goal = {
-    <e1:ShiftExpression> <location:@L> <op:ShiftOp> <e2:ArithmeticExpression> <end_location:@R> => ast::Expr {
+    <location:@L> <e1:ShiftExpression> <op:ShiftOp> <e2:ArithmeticExpression> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
@@ -935,7 +935,7 @@ AddOp: ast::Operator = {
 
 Term = TermAs<ExprGoal>;
 TermAs<Goal>: Goal = {
-    <a:Term> <location:@L> <op:MulOp> <b:Factor> <end_location:@R> => ast::Expr {
+    <location:@L> <a:Term> <op:MulOp> <b:Factor> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
@@ -971,7 +971,7 @@ UnaryOp: ast::Unaryop = {
 
 Power = PowerAs<ExprGoal>;
 PowerAs<Goal>: Goal = {
-    <e:AtomExpr> <location:@L> "**" <b:Factor> <end_location:@R> => ast::Expr {
+    <location:@L> <e:AtomExpr> "**" <b:Factor> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap
index f305878e..df154a37 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap
@@ -78,7 +78,7 @@ expression: parse_ast
                     body: Located {
                         location: Location {
                             row: 1,
-                            column: 15,
+                            column: 13,
                         },
                         end_location: Some(
                             Location {

From 572bbda60bb32c21f37efe088387a6690b4b09f3 Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Thu, 22 Dec 2022 23:27:36 +0900
Subject: [PATCH 330/459] Fix is_local

---
 codegen/src/symboltable.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 3cb2939c..3182f3d3 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -150,7 +150,7 @@ impl Symbol {
     }
 
     pub fn is_local(&self) -> bool {
-        self.scope == SymbolScope::Local
+        matches!(self.scope, SymbolScope::Local | SymbolScope::Cell)
     }
 
     pub fn is_bound(&self) -> bool {

From b20571ad0fb4b6a0d60fa146751d238cda8df527 Mon Sep 17 00:00:00 2001
From: Anders Kaseorg <andersk@mit.edu>
Date: Sun, 18 Dec 2022 17:32:31 -0800
Subject: [PATCH 331/459] Simplify parenthesized context manager parsing with
 LALRPOP conditions

Signed-off-by: Anders Kaseorg <andersk@mit.edu>
---
 parser/python.lalrpop | 313 ++++++++++++++++++++----------------------
 parser/src/with.rs    | 134 ------------------
 2 files changed, 149 insertions(+), 298 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index c1f6ee07..f586f0a7 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -11,7 +11,6 @@ use crate::{
     context::set_context,
     string::parse_strings,
     token::StringKind,
-    with::{ExprOrWithitems, TupleOrWithitems},
 };
 use num_bigint::BigInt;
 
@@ -129,7 +128,7 @@ ExpressionStatement: ast::Stmt = {
             },
         }
     },
-    <location:@L> <target:Test> ":" <annotation:Test> <rhs:AssignSuffix?> <end_location:@R> => {
+    <location:@L> <target:Test<"all">> ":" <annotation:Test<"all">> <rhs:AssignSuffix?> <end_location:@R> => {
         let simple = matches!(target.node, ast::ExprKind::Name { .. });
         ast::Stmt {
             custom: (),
@@ -161,7 +160,7 @@ TestOrStarExprList: ast::Expr = {
 };
 
 TestOrStarExpr: ast::Expr = {
-    Test,
+    Test<"all">,
     StarExpr,
 };
 
@@ -170,12 +169,6 @@ TestOrStarNamedExpr: ast::Expr = {
     StarExpr,
 };
 
-TestOrStarNamedExprOrWithitem: (ast::Expr, Option<Box<ast::Expr>>) = {
-    <e:NamedExpressionTest> => (e, None),
-    <e:StarExpr> => (e, None),
-    <e:Test> "as" <v:Expression> => (e, Some(Box::new(v))),
-}
-
 AugAssign: ast::Operator = {
     "+=" => ast::Operator::Add,
     "-=" => ast::Operator::Sub,
@@ -237,7 +230,7 @@ RaiseStatement: ast::Stmt = {
             node: ast::StmtKind::Raise { exc: None, cause: None },
         }
     },
-    <location:@L> "raise" <t:Test> <c:("from" Test)?> <end_location:@R> => {
+    <location:@L> "raise" <t:Test<"all">> <c:("from" Test<"all">)?> <end_location:@R> => {
         ast::Stmt {
             custom: (),
             location,
@@ -336,7 +329,7 @@ NonlocalStatement: ast::Stmt = {
 };
 
 AssertStatement: ast::Stmt = {
-    <location:@L> "assert" <test:Test> <msg: ("," Test)?> <end_location:@R> => {
+    <location:@L> "assert" <test:Test<"all">> <msg: ("," Test<"all">)?> <end_location:@R> => {
         ast::Stmt {
             custom: (),
             location,
@@ -473,7 +466,7 @@ TryStatement: ast::Stmt = {
 };
 
 ExceptClause: ast::Excepthandler = {
-    <location:@L> "except" <typ:Test?> ":" <body:Suite> => {
+    <location:@L> "except" <typ:Test<"all">?> ":" <body:Suite> => {
         let end_location = body.last().unwrap().end_location.unwrap();
         ast::Excepthandler::new(
             location,
@@ -485,7 +478,7 @@ ExceptClause: ast::Excepthandler = {
             },
         )
     },
-    <location:@L> "except" <x:(Test "as" Identifier)> ":" <body:Suite> => {
+    <location:@L> "except" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
         let end_location = body.last().unwrap().end_location.unwrap();
         ast::Excepthandler::new(
             location,
@@ -512,34 +505,34 @@ WithStatement: ast::Stmt = {
     },
 };
 
-// These are only used for their types as macro parameters
-ExprGoal: ast::Expr = {};
-ExprOrWithitemsGoal: ExprOrWithitems = {};
-
 WithItems: Vec<ast::Withitem> = {
-    <items:TestAs<ExprOrWithitemsGoal>> =>? items.try_into(),
-    <first:TestAs<ExprOrWithitemsGoal>> "as" <vars:Expression> =>? {
-        let optional_vars = Some(Box::new(set_context(vars, ast::ExprContext::Store)));
-        let context_expr = first.try_into()?;
-        Ok(vec![ast::Withitem { context_expr, optional_vars }])
-    },
-    <first:TestAs<ExprOrWithitemsGoal>> <n:("as" Expression)?> "," <mut items:OneOrMore<WithItem>> =>? {
-        let optional_vars = n.map(|val| Box::new(set_context(val.1, ast::ExprContext::Store)));
-        let context_expr = first.try_into()?;
-        items.insert(0, ast::Withitem { context_expr, optional_vars });
-        Ok(items)
+    "(" <WithItemsNoAs> ","? ")",
+    "(" <left:(<WithItemsNoAs> ",")?> <mid:WithItem<"as">> <right:("," <WithItem<"all">>)*> ","? ")" => {
+        left.into_iter().flatten().chain([mid]).chain(right).collect()
+    },
+    <WithItem<"no-withitems">> => vec![<>],
+    <item:WithItem<"all">> <items:("," <WithItem<"all">>)+> => {
+        [item].into_iter().chain(items).collect()
     }
 };
 
-WithItem: ast::Withitem = {
-    <context_expr:Test> <n:("as" Expression)?> => {
-        let optional_vars = n.map(|val| Box::new(set_context(val.1, ast::ExprContext::Store)));
+#[inline]
+WithItemsNoAs: Vec<ast::Withitem> = {
+    <OneOrMore<NamedExpressionTest>> => {
+        <>.into_iter().map(|context_expr| ast::Withitem { context_expr, optional_vars: None }).collect()
+    },
+}
+
+WithItem<Goal>: ast::Withitem = {
+    <Test<Goal>> if Goal != "as" => ast::Withitem { context_expr: <>, optional_vars: None },
+    <context_expr:Test<"all">> "as" <vars:Expression<"all">> => {
+        let optional_vars = Some(Box::new(set_context(vars, ast::ExprContext::Store)));
         ast::Withitem { context_expr, optional_vars }
     },
 };
 
 FuncDef: ast::Stmt = {
-    <decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test)?> ":" <body:Suite>  => {
+    <decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test<"all">)?> ":" <body:Suite>  => {
         let args = Box::new(args);
         let returns = r.map(|x| Box::new(x.1));
         let end_location = body.last().unwrap().end_location.unwrap();
@@ -643,7 +636,7 @@ ParameterDefs<ArgType>: (Vec<(ast::Arg, Option<ast::Expr>)>, Vec<(ast::Arg, Opti
 
 ParameterDef<ArgType>: (ast::Arg, Option<ast::Expr>) = {
     <i:ArgType> => (i, None),
-    <i:ArgType> "=" <e:Test> => (i, Some(e)),
+    <i:ArgType> "=" <e:Test<"all">> => (i, Some(e)),
 };
 
 UntypedParameter: ast::Arg = {
@@ -655,7 +648,7 @@ UntypedParameter: ast::Arg = {
 };
 
 TypedParameter: ast::Arg = {
-    <location:@L> <arg:Identifier> <a:(":" Test)?> <end_location:@R> => {
+    <location:@L> <arg:Identifier> <a:(":" Test<"all">)?> <end_location:@R> => {
         let annotation = a.map(|x| Box::new(x.1));
         ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None })
     },
@@ -729,7 +722,7 @@ YieldExpr: ast::Expr = {
         custom: (),
         node: ast::ExprKind::Yield { value: value.map(Box::new) }
     },
-    <location:@L> "yield" "from" <e:Test> <end_location:@R> => ast::Expr {
+    <location:@L> "yield" "from" <e:Test<"all">> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
@@ -737,9 +730,8 @@ YieldExpr: ast::Expr = {
     },
 };
 
-Test = TestAs<ExprGoal>;
-TestAs<Goal>: Goal = {
-    <body:OrTest> <location:@L> "if" <test:OrTest> "else" <orelse:Test> <end_location:@R> => ast::Expr {
+Test<Goal>: ast::Expr = {
+    <body:OrTest<"all">> <location:@L> "if" <test:OrTest<"all">> "else" <orelse:Test<"all">> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
@@ -748,13 +740,13 @@ TestAs<Goal>: Goal = {
             body: Box::new(body),
             orelse: Box::new(orelse),
         }
-    }.into(),
-    OrTestAs<Goal>,
-    <e:LambdaDef> => e.into(),
+    },
+    OrTest<Goal>,
+    LambdaDef,
 };
 
 NamedExpressionTest: ast::Expr = {
-    <location:@L> <left: (Identifier ":=")?> <right:Test> <end_location:@R> => {
+    <location:@L> <left: (Identifier ":=")?> <right:Test<"all">> <end_location:@R> => {
         if let Some(l) = left {
             ast::Expr {
                 location,
@@ -776,7 +768,7 @@ NamedExpressionTest: ast::Expr = {
 }
 
 LambdaDef: ast::Expr = {
-    <location:@L> "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test> <end_location:@R> => {
+    <location:@L> "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test<"all">> <end_location:@R> => {
         let p = p.unwrap_or_else(|| {
             ast::Arguments {
                 posonlyargs: vec![],
@@ -800,9 +792,8 @@ LambdaDef: ast::Expr = {
     }
 }
 
-OrTest = OrTestAs<ExprGoal>;
-OrTestAs<Goal>: Goal = {
-    <location:@L> <e1:AndTest> <e2:("or" AndTest)+> <end_location:@R> => {
+OrTest<Goal>: ast::Expr = {
+    <location:@L> <e1:AndTest<"all">> <e2:("or" AndTest<"all">)+> <end_location:@R> => {
         let mut values = vec![e1];
         values.extend(e2.into_iter().map(|e| e.1));
         ast::Expr {
@@ -810,14 +801,13 @@ OrTestAs<Goal>: Goal = {
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values }
-        }.into()
+        }
     },
-    AndTestAs<Goal>,
+    AndTest<Goal>,
 };
 
-AndTest = AndTestAs<ExprGoal>;
-AndTestAs<Goal>: Goal = {
-    <location:@L> <e1:NotTest> <e2:("and" NotTest)+> <end_location:@R> => {
+AndTest<Goal>: ast::Expr = {
+    <location:@L> <e1:NotTest<"all">> <e2:("and" NotTest<"all">)+> <end_location:@R> => {
         let mut values = vec![e1];
         values.extend(e2.into_iter().map(|e| e.1));
         ast::Expr {
@@ -825,34 +815,32 @@ AndTestAs<Goal>: Goal = {
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values }
-        }.into()
+        }
     },
-    NotTestAs<Goal>,
+    NotTest<Goal>,
 };
 
-NotTest = NotTestAs<ExprGoal>;
-NotTestAs<Goal>: Goal = {
-    <location:@L> "not" <e:NotTest> <end_location:@R> => ast::Expr {
+NotTest<Goal>: ast::Expr = {
+    <location:@L> "not" <e:NotTest<"all">> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::UnaryOp { operand: Box::new(e), op: ast::Unaryop::Not }
-    }.into(),
-    ComparisonAs<Goal>,
+    },
+    Comparison<Goal>,
 };
 
-Comparison = ComparisonAs<ExprGoal>;
-ComparisonAs<Goal>: Goal = {
-    <location:@L> <left:Expression> <comparisons:(CompOp Expression)+> <end_location:@R> => {
+Comparison<Goal>: ast::Expr = {
+    <location:@L> <left:Expression<"all">> <comparisons:(CompOp Expression<"all">)+> <end_location:@R> => {
         let (ops, comparators) = comparisons.into_iter().unzip();
         ast::Expr {
             location,
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::Compare { left: Box::new(left), ops, comparators }
-        }.into()
+        }
     },
-    ExpressionAs<Goal>,
+    Expression<Goal>,
 };
 
 CompOp: ast::Cmpop = {
@@ -868,48 +856,44 @@ CompOp: ast::Cmpop = {
     "is" "not" => ast::Cmpop::IsNot,
 };
 
-Expression = ExpressionAs<ExprGoal>;
-ExpressionAs<Goal>: Goal = {
-    <location:@L> <e1:Expression> "|" <e2:XorExpression> <end_location:@R> => ast::Expr {
+Expression<Goal>: ast::Expr = {
+    <location:@L> <e1:Expression<"all">> "|" <e2:XorExpression<"all">> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) }
-    }.into(),
-    XorExpressionAs<Goal>,
+    },
+    XorExpression<Goal>,
 };
 
-XorExpression = XorExpressionAs<ExprGoal>;
-XorExpressionAs<Goal>: Goal = {
-     <location:@L> <e1:XorExpression> "^" <e2:AndExpression> <end_location:@R> => ast::Expr {
+XorExpression<Goal>: ast::Expr = {
+    <location:@L> <e1:XorExpression<"all">> "^" <e2:AndExpression<"all">> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) }
-    }.into(),
-    AndExpressionAs<Goal>,
+    },
+    AndExpression<Goal>,
 };
 
-AndExpression = AndExpressionAs<ExprGoal>;
-AndExpressionAs<Goal>: Goal = {
-    <location:@L> <e1:AndExpression> "&" <e2:ShiftExpression> <end_location:@R> => ast::Expr {
+AndExpression<Goal>: ast::Expr = {
+    <location:@L> <e1:AndExpression<"all">> "&" <e2:ShiftExpression<"all">> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) }
-    }.into(),
-    ShiftExpressionAs<Goal>,
+    },
+    ShiftExpression<Goal>,
 };
 
-ShiftExpression = ShiftExpressionAs<ExprGoal>;
-ShiftExpressionAs<Goal>: Goal = {
-    <location:@L> <e1:ShiftExpression> <op:ShiftOp> <e2:ArithmeticExpression> <end_location:@R> => ast::Expr {
+ShiftExpression<Goal>: ast::Expr = {
+    <location:@L> <e1:ShiftExpression<"all">> <op:ShiftOp> <e2:ArithmeticExpression<"all">> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) }
-    }.into(),
-    ArithmeticExpressionAs<Goal>,
+    },
+    ArithmeticExpression<Goal>,
 };
 
 ShiftOp: ast::Operator = {
@@ -917,15 +901,14 @@ ShiftOp: ast::Operator = {
     ">>" => ast::Operator::RShift,
 };
 
-ArithmeticExpression = ArithmeticExpressionAs<ExprGoal>;
-ArithmeticExpressionAs<Goal>: Goal = {
-    <location:@L> <a:ArithmeticExpression> <op:AddOp> <b:Term> <end_location:@R> => ast::Expr {
+ArithmeticExpression<Goal>: ast::Expr = {
+    <location:@L> <a:ArithmeticExpression<"all">> <op:AddOp> <b:Term<"all">> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
-    }.into(),
-    TermAs<Goal>,
+    },
+    Term<Goal>,
 };
 
 AddOp: ast::Operator = {
@@ -933,15 +916,14 @@ AddOp: ast::Operator = {
     "-" => ast::Operator::Sub,
 };
 
-Term = TermAs<ExprGoal>;
-TermAs<Goal>: Goal = {
-    <location:@L> <a:Term> <op:MulOp> <b:Factor> <end_location:@R> => ast::Expr {
+Term<Goal>: ast::Expr = {
+    <location:@L> <a:Term<"all">> <op:MulOp> <b:Factor<"all">> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
-    }.into(),
-    FactorAs<Goal>,
+    },
+    Factor<Goal>,
 };
 
 MulOp: ast::Operator = {
@@ -952,15 +934,14 @@ MulOp: ast::Operator = {
     "@" => ast::Operator::MatMult,
 };
 
-Factor = FactorAs<ExprGoal>;
-FactorAs<Goal>: Goal = {
-    <location:@L> <op:UnaryOp> <e:Factor> <end_location:@R>  => ast::Expr {
+Factor<Goal>: ast::Expr = {
+    <location:@L> <op:UnaryOp> <e:Factor<"all">> <end_location:@R>  => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::UnaryOp { operand: Box::new(e), op }
-    }.into(),
-    PowerAs<Goal>,
+    },
+    Power<Goal>,
 };
 
 UnaryOp: ast::Unaryop = {
@@ -969,53 +950,50 @@ UnaryOp: ast::Unaryop = {
     "~" => ast::Unaryop::Invert,
 };
 
-Power = PowerAs<ExprGoal>;
-PowerAs<Goal>: Goal = {
-    <location:@L> <e:AtomExpr> "**" <b:Factor> <end_location:@R> => ast::Expr {
+Power<Goal>: ast::Expr = {
+    <location:@L> <e:AtomExpr<"all">> "**" <b:Factor<"all">> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) }
-    }.into(),
-    AtomExprAs<Goal>,
+    },
+    AtomExpr<Goal>,
 };
 
-AtomExpr = AtomExprAs<ExprGoal>;
-AtomExprAs<Goal>: Goal = {
-    <location:@L> "await" <atom:AtomExpr2> <end_location:@R> => {
+AtomExpr<Goal>: ast::Expr = {
+    <location:@L> "await" <atom:AtomExpr2<"all">> <end_location:@R> => {
         ast::Expr {
             location,
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::Await { value: Box::new(atom) }
-        }.into()
+        }
     },
-    AtomExpr2As<Goal>,
+    AtomExpr2<Goal>,
 }
 
-AtomExpr2 = AtomExpr2As<ExprGoal>;
-AtomExpr2As<Goal>: Goal = {
-    AtomAs<Goal>,
-    <location:@L> <f:AtomExpr2> "(" <a:ArgumentList> ")" <end_location:@R> => {
+AtomExpr2<Goal>: ast::Expr = {
+    Atom<Goal>,
+    <location:@L> <f:AtomExpr2<"all">> "(" <a:ArgumentList> ")" <end_location:@R> => {
         ast::Expr {
             location,
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::Call { func: Box::new(f), args: a.args, keywords: a.keywords }
-        }.into()
+        }
     },
-    <location:@L> <e:AtomExpr2> "[" <s:SubscriptList> "]" <end_location:@R> => ast::Expr {
+    <location:@L> <e:AtomExpr2<"all">> "[" <s:SubscriptList> "]" <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Subscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load }
-    }.into(),
-    <location:@L> <e:AtomExpr2> "." <attr:Identifier> <end_location:@R> => ast::Expr {
+    },
+    <location:@L> <e:AtomExpr2<"all">> "." <attr:Identifier> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Attribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load }
-    }.into(),
+    },
 };
 
 SubscriptList: ast::Expr = {
@@ -1039,8 +1017,8 @@ SubscriptList: ast::Expr = {
 };
 
 Subscript: ast::Expr = {
-    Test,
-    <e1:Test?> <location:@L> ":" <e2:Test?> <e3:SliceOp?> <end_location:@R> => {
+    Test<"all">,
+    <e1:Test<"all">?> <location:@L> ":" <e2:Test<"all">?> <e3:SliceOp?> <end_location:@R> => {
         let lower = e1.map(Box::new);
         let upper = e2.map(Box::new);
         let step = e3.flatten().map(Box::new);
@@ -1054,24 +1032,23 @@ Subscript: ast::Expr = {
 };
 
 SliceOp: Option<ast::Expr> = {
-    <location:@L> ":" <e:Test?> => e,
+    <location:@L> ":" <e:Test<"all">?> => e,
 }
 
-Atom = AtomAs<ExprGoal>;
-AtomAs<Goal>: Goal = {
-    <location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?.into()),
+Atom<Goal>: ast::Expr = {
+    <location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?),
     <location:@L> <value:Constant> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Constant { value, kind: None }
-    }.into(),
+    },
     <location:@L> <name:Identifier> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load }
-    }.into(),
+    },
     <location:@L> "[" <e:ListLiteralValues?> "]"<end_location:@R>  => {
         let elts = e.unwrap_or_default();
         ast::Expr {
@@ -1079,7 +1056,7 @@ AtomAs<Goal>: Goal = {
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::List { elts, ctx: ast::ExprContext::Load }
-        }.into()
+        }
     },
     <location:@L> "[" <elt:TestOrStarNamedExpr> <generators:CompFor> "]" <end_location:@R> => {
         ast::Expr {
@@ -1087,40 +1064,48 @@ AtomAs<Goal>: Goal = {
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::ListComp { elt: Box::new(elt), generators }
-        }.into()
-    },
-    <location:@L> "(" <items:OneOrMore<TestOrStarNamedExprOrWithitem>> <trailing_comma:","?> ")" <end_location:@R> =>? {
-        if items.len() == 1 && items[0].1.is_none() && trailing_comma.is_none() {
-            match items[0].0.node {
-                ast::ExprKind::Starred { .. } => {
-                    Err(LexicalError{
-                        error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
-                        location: items[0].0.location,
-                    }.into())
-                }
-                _ => {
-                    Ok(items.into_iter().next().unwrap().0.into())
-                }
-            }
+        }
+    },
+    <location:@L> "(" <elts:OneOrMore<NamedExpressionTest>> <trailing_comma:","?> ")" <end_location:@R> if Goal != "no-withitems" => {
+        if elts.len() == 1 && trailing_comma.is_none() {
+            elts.into_iter().next().unwrap()
         } else {
-            TupleOrWithitems { location, end_location, items }.try_into()
+            ast::Expr::new(
+                location,
+                end_location,
+                ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load },
+            )
+        }
+    },
+    <location:@L> "(" <left:(<OneOrMore<NamedExpressionTest>> ",")?> <mid:StarExpr> <right:("," <TestOrStarNamedExpr>)*> <trailing_comma:","?> ")" <end_location:@R> =>? {
+        if left.is_none() && right.is_empty() && trailing_comma.is_none() {
+            Err(LexicalError{
+                error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
+                location: mid.location,
+            })?
         }
+        let elts = left.into_iter().flatten().chain([mid]).chain(right).collect();
+        Ok(ast::Expr::new(
+            location,
+            end_location,
+            ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load },
+        ))
     },
     <location:@L> "(" ")" <end_location:@R> => ast::Expr::new(
         location,
         end_location,
         ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load }
-    ).into(),
-    "(" <e:YieldExpr> ")" => e.into(),
+    ),
+    "(" <e:YieldExpr> ")" => e,
     <location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" <end_location:@R> => {
         ast::Expr {
             location,
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::GeneratorExp { elt: Box::new(elt), generators }
-        }.into()
+        }
     },
-    "(" <location:@L> "**" <e:Expression> ")" <end_location:@R> =>? {
+    "(" <location:@L> "**" <e:Expression<"all">> ")" <end_location:@R> =>? {
         Err(LexicalError{
             error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()),
             location: location,
@@ -1170,7 +1155,7 @@ AtomAs<Goal>: Goal = {
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::Dict { keys, values }
-        }.into()
+        }
     },
     <location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" <end_location:@R> => {
         ast::Expr {
@@ -1182,26 +1167,26 @@ AtomAs<Goal>: Goal = {
                 value: Box::new(e1.1),
                 generators,
             }
-        }.into()
+        }
     },
     <location:@L> "{" <elts:SetLiteralValues> "}" <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
         node: ast::ExprKind::Set { elts }
-    }.into(),
-    <location:@L> "{" <elt:Test> <generators:CompFor> "}" <end_location:@R> => {
+    },
+    <location:@L> "{" <elt:Test<"all">> <generators:CompFor> "}" <end_location:@R> => {
         ast::Expr {
             location,
             end_location: Some(end_location),
             custom: (),
             node: ast::ExprKind::SetComp { elt: Box::new(elt), generators }
-        }.into()
+        }
     },
-    <location:@L> "True" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: true.into(), kind: None }).into(),
-    <location:@L> "False" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: false.into(), kind: None }).into(),
-    <location:@L> "None" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }).into(),
-    <location:@L> "..." <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }).into(),
+    <location:@L> "True" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: true.into(), kind: None }),
+    <location:@L> "False" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: false.into(), kind: None }),
+    <location:@L> "None" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }),
+    <location:@L> "..." <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }),
 };
 
 ListLiteralValues: Vec<ast::Expr> = {
@@ -1213,12 +1198,12 @@ DictLiteralValues: Vec<(Option<Box<ast::Expr>>, ast::Expr)> = {
 };
 
 DictEntry: (ast::Expr, ast::Expr) = {
-    <e1: Test> ":" <e2: Test> => (e1, e2),
+    <e1: Test<"all">> ":" <e2: Test<"all">> => (e1, e2),
 };
 
 DictElement: (Option<Box<ast::Expr>>, ast::Expr) = {
     <e:DictEntry> => (Some(Box::new(e.0)), e.1),
-    "**" <e:Expression> => (None, e),
+    "**" <e:Expression<"all">> => (None, e),
 };
 
 SetLiteralValues: Vec<ast::Expr> = {
@@ -1226,7 +1211,7 @@ SetLiteralValues: Vec<ast::Expr> = {
 };
 
 ExpressionOrStarExpression = {
-    Expression,
+    Expression<"all">,
     StarExpr
 };
 
@@ -1264,7 +1249,7 @@ GenericList<Element>: ast::Expr = {
 
 // Test
 StarExpr: ast::Expr = {
-    <location:@L> "*" <e:Expression> <end_location:@R> => ast::Expr {
+    <location:@L> "*" <e:Expression<"all">> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
@@ -1276,7 +1261,7 @@ StarExpr: ast::Expr = {
 CompFor: Vec<ast::Comprehension> = <c:SingleForComprehension+> => c;
 
 SingleForComprehension: ast::Comprehension = {
-    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest> <ifs:ComprehensionIf*> <end_location:@R> => {
+    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest<"all">> <ifs:ComprehensionIf*> <end_location:@R> => {
         let is_async = is_async.is_some();
         ast::Comprehension {
             target: set_context(target, ast::ExprContext::Store),
@@ -1287,7 +1272,7 @@ SingleForComprehension: ast::Comprehension = {
     }
 };
 
-ExpressionNoCond: ast::Expr = OrTest;
+ExpressionNoCond: ast::Expr = OrTest<"all">;
 ComprehensionIf: ast::Expr = "if" <c:ExpressionNoCond> => c;
 
 ArgumentList: ArgumentList = {
@@ -1313,8 +1298,8 @@ FunctionArgument: (Option<(ast::Location, ast::Location, Option<String>)>, ast::
         };
         (None, expr)
     },
-    <location:@L> <i:Identifier> "=" <e:Test> <end_location:@R> => (Some((location, end_location, Some(i))), e),
-    <location:@L> "*" <e:Test> <end_location:@R> => {
+    <location:@L> <i:Identifier> "=" <e:Test<"all">> <end_location:@R> => (Some((location, end_location, Some(i))), e),
+    <location:@L> "*" <e:Test<"all">> <end_location:@R> => {
         let expr = ast::Expr::new(
             location,
             end_location,
@@ -1322,7 +1307,7 @@ FunctionArgument: (Option<(ast::Location, ast::Location, Option<String>)>, ast::
         );
         (None, expr)
     },
-    <location:@L> "**" <e:Test> <end_location:@R> => (Some((location, end_location, None)), e),
+    <location:@L> "**" <e:Test<"all">> <end_location:@R> => (Some((location, end_location, None)), e),
 };
 
 #[inline]
diff --git a/parser/src/with.rs b/parser/src/with.rs
index 5f018c6b..b05fb373 100644
--- a/parser/src/with.rs
+++ b/parser/src/with.rs
@@ -1,137 +1,3 @@
-//! Intermediate types for `with` statement cover grammar.
-//!
-//! When we start parsing a `with` statement, we don't initially know
-//! whether we're looking at a tuple or a Python 3.9+ parenthesized
-//! collection of contexts:
-//!
-//! ```python
-//! with (a, b, c) as t:  # tuple
-//! with (a, b, c):  # withitems
-//! ```
-//!
-//! Since LALRPOP requires us to commit to an output type before we
-//! have enough information to decide, we build a cover grammar that's
-//! convertible either way.  This module contains the necessary
-//! intermediate data types.
-
-use crate::ast::{self, Location};
-use crate::context;
-use crate::error::{LexicalError, LexicalErrorType};
-use crate::token::Tok;
-use lalrpop_util::ParseError as LalrpopError;
-
-/// Represents a parenthesized collection that we might later convert
-/// to a tuple or to `with` items.
-///
-/// It can be converted to either `Expr` or `ExprOrWithitems` with
-/// `.try_into()`.  The `Expr` conversion will fail if any `as`
-/// variables are present.  The `ExprOrWithitems` conversion cannot
-/// fail (but we need it to have the same interface so we can use
-/// LALRPOP macros to declare the cover grammar without much code
-/// duplication).
-pub struct TupleOrWithitems {
-    pub location: Location,
-    pub end_location: Location,
-    pub items: Vec<(ast::Expr, Option<Box<ast::Expr>>)>,
-}
-
-impl TryFrom<TupleOrWithitems> for ast::Expr {
-    type Error = LalrpopError<Location, Tok, LexicalError>;
-    fn try_from(tuple_or_withitems: TupleOrWithitems) -> Result<ast::Expr, Self::Error> {
-        Ok(ast::Expr {
-            location: tuple_or_withitems.location,
-            end_location: Some(tuple_or_withitems.end_location),
-            custom: (),
-            node: ast::ExprKind::Tuple {
-                elts: tuple_or_withitems
-                    .items
-                    .into_iter()
-                    .map(|(expr, optional_vars)| {
-                        if let Some(vars) = optional_vars {
-                            Err(LexicalError {
-                                error: LexicalErrorType::OtherError(
-                                    "cannot use 'as' here".to_string(),
-                                ),
-                                location: vars.location,
-                            })?
-                        }
-                        Ok(expr)
-                    })
-                    .collect::<Result<Vec<ast::Expr>, Self::Error>>()?,
-                ctx: ast::ExprContext::Load,
-            },
-        })
-    }
-}
-
-impl TryFrom<TupleOrWithitems> for ExprOrWithitems {
-    type Error = LalrpopError<Location, Tok, LexicalError>;
-    fn try_from(items: TupleOrWithitems) -> Result<ExprOrWithitems, Self::Error> {
-        Ok(ExprOrWithitems::TupleOrWithitems(items))
-    }
-}
-
-/// Represents either a non-tuple expression, or a parenthesized
-/// collection that we might later convert to a tuple or to `with`
-/// items.
-///
-/// It can be constructed from an `Expr` with `.into()`.  (The same
-/// interface can be used to convert an `Expr` into itself, which is
-/// also important for our LALRPOP macro setup.)
-///
-/// It can be converted to either `Expr` or `Vec<Withitem>` with
-/// `.try_into()`.  The `Expr` conversion will fail if any `as`
-/// clauses are present.  The `Vec<Withitem>` conversion will fail if
-/// both `as` clauses and starred expressions are present.
-pub enum ExprOrWithitems {
-    Expr(ast::Expr),
-    TupleOrWithitems(TupleOrWithitems),
-}
-
-impl From<ast::Expr> for ExprOrWithitems {
-    fn from(expr: ast::Expr) -> ExprOrWithitems {
-        ExprOrWithitems::Expr(expr)
-    }
-}
-
-impl TryFrom<ExprOrWithitems> for ast::Expr {
-    type Error = LalrpopError<Location, Tok, LexicalError>;
-    fn try_from(expr_or_withitems: ExprOrWithitems) -> Result<ast::Expr, Self::Error> {
-        match expr_or_withitems {
-            ExprOrWithitems::Expr(expr) => Ok(expr),
-            ExprOrWithitems::TupleOrWithitems(items) => items.try_into(),
-        }
-    }
-}
-
-impl TryFrom<ExprOrWithitems> for Vec<ast::Withitem> {
-    type Error = LalrpopError<Location, Tok, LexicalError>;
-    fn try_from(expr_or_withitems: ExprOrWithitems) -> Result<Vec<ast::Withitem>, Self::Error> {
-        match expr_or_withitems {
-            ExprOrWithitems::TupleOrWithitems(tuple_or_withitems)
-                if !tuple_or_withitems.items.iter().any(|(context_expr, _)| {
-                    matches!(context_expr.node, ast::ExprKind::Starred { .. })
-                }) =>
-            {
-                Ok(tuple_or_withitems
-                    .items
-                    .into_iter()
-                    .map(|(context_expr, optional_vars)| ast::Withitem {
-                        context_expr,
-                        optional_vars: optional_vars.map(|expr| {
-                            Box::new(context::set_context(*expr, ast::ExprContext::Store))
-                        }),
-                    })
-                    .collect())
-            }
-            _ => Ok(vec![ast::Withitem {
-                context_expr: expr_or_withitems.try_into()?,
-                optional_vars: None,
-            }]),
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use crate::parser::parse_program;

From 6fc0d42d4a536b0d20812204ea7f409761194bda Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sat, 24 Dec 2022 13:38:33 +0900
Subject: [PATCH 332/459] Fix IfExp location

---
 parser/python.lalrpop                                         | 2 +-
 ..._parser__tests__parse_if_else_generator_comprehension.snap | 2 +-
 .../rustpython_parser__with__tests__with_statement.snap       | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index f586f0a7..85dea576 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -731,7 +731,7 @@ YieldExpr: ast::Expr = {
 };
 
 Test<Goal>: ast::Expr = {
-    <body:OrTest<"all">> <location:@L> "if" <test:OrTest<"all">> "else" <orelse:Test<"all">> <end_location:@R> => ast::Expr {
+    <location:@L> <body:OrTest<"all">> "if" <test:OrTest<"all">> "else" <orelse:Test<"all">> <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
         custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
index d9762f53..92b85877 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap
@@ -18,7 +18,7 @@ Located {
         elt: Located {
             location: Location {
                 row: 1,
-                column: 3,
+                column: 1,
             },
             end_location: Some(
                 Location {
diff --git a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
index 031d6d83..95ad6f69 100644
--- a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
+++ b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
@@ -343,7 +343,7 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                     context_expr: Located {
                         location: Location {
                             row: 5,
-                            column: 7,
+                            column: 5,
                         },
                         end_location: Some(
                             Location {
@@ -452,7 +452,7 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                     context_expr: Located {
                         location: Location {
                             row: 6,
-                            column: 7,
+                            column: 5,
                         },
                         end_location: Some(
                             Location {

From 56da6c91b716d1eb4a9f90145349b2014dc41db8 Mon Sep 17 00:00:00 2001
From: Anders Kaseorg <andersk@mit.edu>
Date: Sat, 24 Dec 2022 11:21:16 -0800
Subject: [PATCH 333/459] Fix parsing of tuple with named expression as context
 manager
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Because the ‘with’ item grammar disallows named expressions, CPython
parses ‘with (a := 0, b := 1):’ as a tuple rather than two ‘with’
items.

Signed-off-by: Anders Kaseorg <andersk@mit.edu>
---
 parser/python.lalrpop                         |  72 ++++---
 ...n_parser__with__tests__with_statement.snap | 183 ++++++++++--------
 2 files changed, 140 insertions(+), 115 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index f586f0a7..39790180 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -164,6 +164,11 @@ TestOrStarExpr: ast::Expr = {
     StarExpr,
 };
 
+NamedOrStarExpr: ast::Expr = {
+    NamedExpression,
+    StarExpr,
+};
+
 TestOrStarNamedExpr: ast::Expr = {
     NamedExpressionTest,
     StarExpr,
@@ -518,7 +523,7 @@ WithItems: Vec<ast::Withitem> = {
 
 #[inline]
 WithItemsNoAs: Vec<ast::Withitem> = {
-    <OneOrMore<NamedExpressionTest>> => {
+    <OneOrMore<Test<"all">>> => {
         <>.into_iter().map(|context_expr| ast::Withitem { context_expr, optional_vars: None }).collect()
     },
 }
@@ -746,26 +751,27 @@ Test<Goal>: ast::Expr = {
 };
 
 NamedExpressionTest: ast::Expr = {
-    <location:@L> <left: (Identifier ":=")?> <right:Test<"all">> <end_location:@R> => {
-        if let Some(l) = left {
-            ast::Expr {
-                location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::ExprKind::NamedExpr {
-                    target: Box::new(ast::Expr::new(
-                        location,
-                        end_location,
-                        ast::ExprKind::Name { id: l.0, ctx: ast::ExprContext::Store },
-                    )),
-                    value: Box::new(right),
-                }
+    NamedExpression,
+    Test<"all">,
+}
+
+NamedExpression: ast::Expr = {
+    <location:@L> <id:Identifier> ":=" <value:Test<"all">> <end_location:@R> => {
+        ast::Expr {
+            location,
+            end_location: Some(end_location),
+            custom: (),
+            node: ast::ExprKind::NamedExpr {
+                target: Box::new(ast::Expr::new(
+                    location,
+                    end_location,
+                    ast::ExprKind::Name { id, ctx: ast::ExprContext::Store },
+                )),
+                value: Box::new(value),
             }
-        } else {
-            right
         }
-    }
-}
+    },
+};
 
 LambdaDef: ast::Expr = {
     <location:@L> "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test<"all">> <end_location:@R> => {
@@ -1066,7 +1072,7 @@ Atom<Goal>: ast::Expr = {
             node: ast::ExprKind::ListComp { elt: Box::new(elt), generators }
         }
     },
-    <location:@L> "(" <elts:OneOrMore<NamedExpressionTest>> <trailing_comma:","?> ")" <end_location:@R> if Goal != "no-withitems" => {
+    <location:@L> "(" <elts:OneOrMore<Test<"all">>> <trailing_comma:","?> ")" <end_location:@R> if Goal != "no-withitems" => {
         if elts.len() == 1 && trailing_comma.is_none() {
             elts.into_iter().next().unwrap()
         } else {
@@ -1077,19 +1083,23 @@ Atom<Goal>: ast::Expr = {
             )
         }
     },
-    <location:@L> "(" <left:(<OneOrMore<NamedExpressionTest>> ",")?> <mid:StarExpr> <right:("," <TestOrStarNamedExpr>)*> <trailing_comma:","?> ")" <end_location:@R> =>? {
+    <location:@L> "(" <left:(<OneOrMore<Test<"all">>> ",")?> <mid:NamedOrStarExpr> <right:("," <TestOrStarNamedExpr>)*> <trailing_comma:","?> ")" <end_location:@R> =>? {
         if left.is_none() && right.is_empty() && trailing_comma.is_none() {
-            Err(LexicalError{
-                error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
-                location: mid.location,
-            })?
+            if matches!(mid.node, ast::ExprKind::Starred { .. }) {
+                Err(LexicalError{
+                    error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
+                    location: mid.location,
+                })?
+            }
+            Ok(mid)
+        } else {
+            let elts = left.into_iter().flatten().chain([mid]).chain(right).collect();
+            Ok(ast::Expr::new(
+                location,
+                end_location,
+                ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load },
+            ))
         }
-        let elts = left.into_iter().flatten().chain([mid]).chain(right).collect();
-        Ok(ast::Expr::new(
-            location,
-            end_location,
-            ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load },
-        ))
     },
     <location:@L> "(" ")" <end_location:@R> => ast::Expr::new(
         location,
diff --git a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
index 031d6d83..7e3da24a 100644
--- a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
+++ b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
@@ -1792,106 +1792,121 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                     context_expr: Located {
                         location: Location {
                             row: 21,
-                            column: 6,
+                            column: 5,
                         },
                         end_location: Some(
                             Location {
                                 row: 21,
-                                column: 12,
+                                column: 21,
                             },
                         ),
                         custom: (),
-                        node: NamedExpr {
-                            target: Located {
-                                location: Location {
-                                    row: 21,
-                                    column: 6,
-                                },
-                                end_location: Some(
-                                    Location {
-                                        row: 21,
-                                        column: 12,
-                                    },
-                                ),
-                                custom: (),
-                                node: Name {
-                                    id: "a",
-                                    ctx: Store,
-                                },
-                            },
-                            value: Located {
-                                location: Location {
-                                    row: 21,
-                                    column: 11,
-                                },
-                                end_location: Some(
-                                    Location {
+                        node: Tuple {
+                            elts: [
+                                Located {
+                                    location: Location {
                                         row: 21,
-                                        column: 12,
+                                        column: 6,
                                     },
-                                ),
-                                custom: (),
-                                node: Constant {
-                                    value: Int(
-                                        0,
+                                    end_location: Some(
+                                        Location {
+                                            row: 21,
+                                            column: 12,
+                                        },
                                     ),
-                                    kind: None,
-                                },
-                            },
-                        },
-                    },
-                    optional_vars: None,
-                },
-                Withitem {
-                    context_expr: Located {
-                        location: Location {
-                            row: 21,
-                            column: 14,
-                        },
-                        end_location: Some(
-                            Location {
-                                row: 21,
-                                column: 20,
-                            },
-                        ),
-                        custom: (),
-                        node: NamedExpr {
-                            target: Located {
-                                location: Location {
-                                    row: 21,
-                                    column: 14,
-                                },
-                                end_location: Some(
-                                    Location {
-                                        row: 21,
-                                        column: 20,
+                                    custom: (),
+                                    node: NamedExpr {
+                                        target: Located {
+                                            location: Location {
+                                                row: 21,
+                                                column: 6,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 21,
+                                                    column: 12,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "a",
+                                                ctx: Store,
+                                            },
+                                        },
+                                        value: Located {
+                                            location: Location {
+                                                row: 21,
+                                                column: 11,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 21,
+                                                    column: 12,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
                                     },
-                                ),
-                                custom: (),
-                                node: Name {
-                                    id: "b",
-                                    ctx: Store,
-                                },
-                            },
-                            value: Located {
-                                location: Location {
-                                    row: 21,
-                                    column: 19,
                                 },
-                                end_location: Some(
-                                    Location {
+                                Located {
+                                    location: Location {
                                         row: 21,
-                                        column: 20,
+                                        column: 14,
                                     },
-                                ),
-                                custom: (),
-                                node: Constant {
-                                    value: Int(
-                                        1,
+                                    end_location: Some(
+                                        Location {
+                                            row: 21,
+                                            column: 20,
+                                        },
                                     ),
-                                    kind: None,
+                                    custom: (),
+                                    node: NamedExpr {
+                                        target: Located {
+                                            location: Location {
+                                                row: 21,
+                                                column: 14,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 21,
+                                                    column: 20,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "b",
+                                                ctx: Store,
+                                            },
+                                        },
+                                        value: Located {
+                                            location: Location {
+                                                row: 21,
+                                                column: 19,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 21,
+                                                    column: 20,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    1,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
                                 },
-                            },
+                            ],
+                            ctx: Load,
                         },
                     },
                     optional_vars: None,

From a5553aea436c6335ca722a0ffe2ceed03a014928 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 25 Dec 2022 09:37:07 +0900
Subject: [PATCH 334/459] Fix slice location

---
 parser/python.lalrpop                         |   2 +-
 parser/src/parser.rs                          |   7 ++
 ...stpython_parser__parser__tests__slice.snap | 115 ++++++++++++++++++
 3 files changed, 123 insertions(+), 1 deletion(-)
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__slice.snap

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 85dea576..04007f0f 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -1018,7 +1018,7 @@ SubscriptList: ast::Expr = {
 
 Subscript: ast::Expr = {
     Test<"all">,
-    <e1:Test<"all">?> <location:@L> ":" <e2:Test<"all">?> <e3:SliceOp?> <end_location:@R> => {
+    <location:@L> <e1:Test<"all">?> ":" <e2:Test<"all">?> <e3:SliceOp?> <end_location:@R> => {
         let lower = e1.map(Box::new);
         let upper = e2.map(Box::new);
         let step = e3.flatten().map(Box::new);
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 27192b6a..705fb991 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -233,4 +233,11 @@ class Foo(A, B):
         let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
+
+    #[test]
+    fn test_slice() {
+        let source = String::from("x[1:2:3]");
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
 }
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__slice.snap b/parser/src/snapshots/rustpython_parser__parser__tests__slice.snap
new file mode 100644
index 00000000..2d404f06
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__slice.snap
@@ -0,0 +1,115 @@
+---
+source: compiler/parser/src/parser.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 1,
+        column: 0,
+    },
+    end_location: Some(
+        Location {
+            row: 1,
+            column: 8,
+        },
+    ),
+    custom: (),
+    node: Subscript {
+        value: Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 1,
+                },
+            ),
+            custom: (),
+            node: Name {
+                id: "x",
+                ctx: Load,
+            },
+        },
+        slice: Located {
+            location: Location {
+                row: 1,
+                column: 2,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 7,
+                },
+            ),
+            custom: (),
+            node: Slice {
+                lower: Some(
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 2,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 3,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                1,
+                            ),
+                            kind: None,
+                        },
+                    },
+                ),
+                upper: Some(
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 4,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 5,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                2,
+                            ),
+                            kind: None,
+                        },
+                    },
+                ),
+                step: Some(
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                3,
+                            ),
+                            kind: None,
+                        },
+                    },
+                ),
+            },
+        },
+        ctx: Load,
+    },
+}

From db91adbcc36100b176935d6770639ec1ea33c42d Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Sun, 25 Dec 2022 11:15:29 +0900
Subject: [PATCH 335/459] Add BOM check

---
 parser/src/lexer.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 04140aa8..a819b4aa 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -236,6 +236,10 @@ where
         lxr.window.slide();
         lxr.window.slide();
         lxr.window.slide();
+        // Check if BOM exists
+        if let Some('\u{feff}') = lxr.window[0] {
+            lxr.window.slide();
+        }
         // Start at top row (=1) left column (=1)
         lxr.location.reset();
         lxr

From 637275fc35df3020f1a7160ff6d95292e52fc192 Mon Sep 17 00:00:00 2001
From: yt2b <reinventing.the.wheels@gmail.com>
Date: Mon, 26 Dec 2022 09:30:12 +0900
Subject: [PATCH 336/459] Fix comment

---
 parser/src/lexer.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index a819b4aa..cb078a25 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -236,7 +236,7 @@ where
         lxr.window.slide();
         lxr.window.slide();
         lxr.window.slide();
-        // Check if BOM exists
+        // TODO: Handle possible mismatch between BOM and explicit encoding declaration.
         if let Some('\u{feff}') = lxr.window[0] {
             lxr.window.slide();
         }

From a0dfa939ff080795725b7628452604ce32708cda Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Tue, 27 Dec 2022 21:54:18 +0900
Subject: [PATCH 337/459] Fix make_tokenizer_located

---
 parser/src/lexer.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index cb078a25..65fb3740 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -154,7 +154,7 @@ pub type LexResult = Result<Spanned, LexicalError>;
 
 #[inline]
 pub fn make_tokenizer(source: &str) -> impl Iterator<Item = LexResult> + '_ {
-    make_tokenizer_located(source, Location::new(0, 0))
+    make_tokenizer_located(source, Location::new(1, 0))
 }
 
 pub fn make_tokenizer_located(
@@ -240,8 +240,6 @@ where
         if let Some('\u{feff}') = lxr.window[0] {
             lxr.window.slide();
         }
-        // Start at top row (=1) left column (=1)
-        lxr.location.reset();
         lxr
     }
 

From b127e7383dced8fc55910a4df0805e141f75ff0b Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Wed, 28 Dec 2022 00:23:25 +0900
Subject: [PATCH 338/459] Refactor parse_formatted_value

---
 parser/src/fstring.rs | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index a28383f1..12b6bf8f 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -26,7 +26,7 @@ impl FStringParser {
         mut chars: iter::Peekable<str::Chars<'a>>,
         nested: u8,
     ) -> Result<(Vec<Expr>, iter::Peekable<str::Chars<'a>>), FStringErrorType> {
-        let mut expression = String::from("{");
+        let mut expression = String::new();
         let mut spec = None;
         let mut delims = Vec::new();
         let mut conversion = ConversionFlag::None;
@@ -67,14 +67,14 @@ impl FStringParser {
                         Some('a') => ConversionFlag::Ascii,
                         Some('r') => ConversionFlag::Repr,
                         Some(_) => {
-                            return Err(if expression[1..].trim().is_empty() {
+                            return Err(if expression.trim().is_empty() {
                                 EmptyExpression
                             } else {
                                 InvalidConversionFlag
                             });
                         }
                         None => {
-                            return Err(if expression[1..].trim().is_empty() {
+                            return Err(if expression.trim().is_empty() {
                                 EmptyExpression
                             } else {
                                 UnclosedLbrace
@@ -84,14 +84,14 @@ impl FStringParser {
 
                     if let Some(&peek) = chars.peek() {
                         if peek != '}' && peek != ':' {
-                            return Err(if expression[1..].trim().is_empty() {
+                            return Err(if expression.trim().is_empty() {
                                 EmptyExpression
                             } else {
                                 UnclosedLbrace
                             });
                         }
                     } else {
-                        return Err(if expression[1..].trim().is_empty() {
+                        return Err(if expression.trim().is_empty() {
                             EmptyExpression
                         } else {
                             UnclosedLbrace
@@ -156,15 +156,14 @@ impl FStringParser {
                     }
                 }
                 '}' => {
-                    if expression[1..].trim().is_empty() {
+                    if expression.trim().is_empty() {
                         return Err(EmptyExpression);
                     }
-                    expression.push(ch);
 
                     let ret = if !self_documenting {
                         vec![self.expr(ExprKind::FormattedValue {
                             value: Box::new(
-                                parse_fstring_expr(&expression[1..expression.len() - 1])
+                                parse_fstring_expr(&expression)
                                     .map_err(|e| InvalidExpression(Box::new(e.error)))?,
                             ),
                             conversion: conversion as _,
@@ -173,9 +172,7 @@ impl FStringParser {
                     } else {
                         vec![
                             self.expr(ExprKind::Constant {
-                                value: Constant::Str(
-                                    expression[1..expression.len() - 1].to_owned() + "=",
-                                ),
+                                value: Constant::Str(expression.to_owned() + "="),
                                 kind: None,
                             }),
                             self.expr(ExprKind::Constant {
@@ -184,7 +181,7 @@ impl FStringParser {
                             }),
                             self.expr(ExprKind::FormattedValue {
                                 value: Box::new(
-                                    parse_fstring_expr(&expression[1..expression.len() - 1])
+                                    parse_fstring_expr(&expression)
                                         .map_err(|e| InvalidExpression(Box::new(e.error)))?,
                                 ),
                                 conversion: (if conversion == ConversionFlag::None && spec.is_none()
@@ -226,7 +223,7 @@ impl FStringParser {
                 }
             }
         }
-        Err(if expression[1..].trim().is_empty() {
+        Err(if expression.trim().is_empty() {
             EmptyExpression
         } else {
             UnclosedLbrace

From 3e3c1417bcd7b8834aacc49d95bfa2acf818f468 Mon Sep 17 00:00:00 2001
From: Anders Kaseorg <andersk@mit.edu>
Date: Tue, 27 Dec 2022 01:48:05 -0800
Subject: [PATCH 339/459] Prohibit starred arguments after double-starred
 arguments
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

CPython prohibits ‘f(**kwargs, *args)’; we should too.

Signed-off-by: Anders Kaseorg <andersk@mit.edu>
---
 parser/src/error.rs    |  7 +++++++
 parser/src/function.rs | 11 ++++++++++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/parser/src/error.rs b/parser/src/error.rs
index 89f53668..47096a54 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -22,6 +22,7 @@ pub enum LexicalErrorType {
     TabsAfterSpaces,
     DefaultArgumentError,
     PositionalArgumentError,
+    UnpackedArgumentError,
     DuplicateKeywordArgumentError,
     UnrecognizedToken { tok: char },
     FStringError(FStringErrorType),
@@ -55,6 +56,12 @@ impl fmt::Display for LexicalErrorType {
             LexicalErrorType::PositionalArgumentError => {
                 write!(f, "positional argument follows keyword argument")
             }
+            LexicalErrorType::UnpackedArgumentError => {
+                write!(
+                    f,
+                    "iterable argument unpacking follows keyword argument unpacking"
+                )
+            }
             LexicalErrorType::UnrecognizedToken { tok } => {
                 write!(f, "Got unexpected token {tok}")
             }
diff --git a/parser/src/function.rs b/parser/src/function.rs
index 0cc934fb..46621130 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -55,6 +55,7 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, Lexi
 
     let mut keyword_names =
         FxHashSet::with_capacity_and_hasher(func_args.len(), Default::default());
+    let mut double_starred = false;
     for (name, value) in func_args {
         match name {
             Some((start, end, name)) => {
@@ -67,6 +68,8 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, Lexi
                     }
 
                     keyword_names.insert(keyword_name.clone());
+                } else {
+                    double_starred = true;
                 }
 
                 keywords.push(ast::Keyword::new(
@@ -76,12 +79,18 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, Lexi
                 ));
             }
             None => {
-                // Allow starred args after keyword arguments.
+                // Allow starred arguments after keyword arguments but
+                // not after double-starred arguments.
                 if !keywords.is_empty() && !is_starred(&value) {
                     return Err(LexicalError {
                         error: LexicalErrorType::PositionalArgumentError,
                         location: value.location,
                     });
+                } else if double_starred {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::UnpackedArgumentError,
+                        location: value.location,
+                    });
                 }
 
                 args.push(value);

From e40e4b21f3359faa3739b65e0f06a848e26fffb9 Mon Sep 17 00:00:00 2001
From: Anders Kaseorg <andersk@mit.edu>
Date: Tue, 27 Dec 2022 18:56:24 -0800
Subject: [PATCH 340/459] Remove duplicate declaration of "from" token

Signed-off-by: Anders Kaseorg <andersk@mit.edu>
---
 parser/python.lalrpop | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 32506eee..a73d8352 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -1428,10 +1428,9 @@ extern {
         "from" => lexer::Tok::From,
         "global" => lexer::Tok::Global,
         "if" => lexer::Tok::If,
+        "import" => lexer::Tok::Import,
         "in" => lexer::Tok::In,
         "is" => lexer::Tok::Is,
-        "import" => lexer::Tok::Import,
-        "from" => lexer::Tok::From,
         "lambda" => lexer::Tok::Lambda,
         "nonlocal" => lexer::Tok::Nonlocal,
         "not" => lexer::Tok::Not,

From 9c99881b7ed3d907a34598147efdabd2e5830f7f Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Thu, 29 Dec 2022 00:08:59 +0900
Subject: [PATCH 341/459] Clean up FStringParser

---
 parser/src/fstring.rs | 128 ++++++++++++++++++++----------------------
 1 file changed, 61 insertions(+), 67 deletions(-)

diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index 12b6bf8f..f92d6425 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -6,14 +6,27 @@ use crate::{
 };
 use std::{iter, mem, str};
 
-struct FStringParser {
+struct FStringParser<'a> {
+    source: iter::Peekable<str::Chars<'a>>,
     str_start: Location,
     str_end: Location,
 }
 
-impl FStringParser {
-    fn new(str_start: Location, str_end: Location) -> Self {
-        Self { str_start, str_end }
+impl<'a> FStringParser<'a> {
+    fn new(source: &'a str, str_start: Location, str_end: Location) -> Self {
+        Self {
+            source: source.chars().peekable(),
+            str_start,
+            str_end,
+        }
+    }
+
+    fn next_char(&mut self) -> Option<char> {
+        self.source.next()
+    }
+
+    fn peek(&mut self) -> Option<&char> {
+        self.source.peek()
     }
 
     #[inline]
@@ -21,11 +34,7 @@ impl FStringParser {
         Expr::new(self.str_start, self.str_end, node)
     }
 
-    fn parse_formatted_value<'a>(
-        &mut self,
-        mut chars: iter::Peekable<str::Chars<'a>>,
-        nested: u8,
-    ) -> Result<(Vec<Expr>, iter::Peekable<str::Chars<'a>>), FStringErrorType> {
+    fn parse_formatted_value(&mut self, nested: u8) -> Result<Vec<Expr>, FStringErrorType> {
         let mut expression = String::new();
         let mut spec = None;
         let mut delims = Vec::new();
@@ -33,36 +42,36 @@ impl FStringParser {
         let mut self_documenting = false;
         let mut trailing_seq = String::new();
 
-        while let Some(ch) = chars.next() {
+        while let Some(ch) = self.next_char() {
             match ch {
                 // can be integrated better with the remainign code, but as a starting point ok
                 // in general I would do here a tokenizing of the fstrings to omit this peeking.
-                '!' if chars.peek() == Some(&'=') => {
+                '!' if self.peek() == Some(&'=') => {
                     expression.push_str("!=");
-                    chars.next();
+                    self.next_char();
                 }
 
-                '=' if chars.peek() == Some(&'=') => {
+                '=' if self.peek() == Some(&'=') => {
                     expression.push_str("==");
-                    chars.next();
+                    self.next_char();
                 }
 
-                '>' if chars.peek() == Some(&'=') => {
+                '>' if self.peek() == Some(&'=') => {
                     expression.push_str(">=");
-                    chars.next();
+                    self.next_char();
                 }
 
-                '<' if chars.peek() == Some(&'=') => {
+                '<' if self.peek() == Some(&'=') => {
                     expression.push_str("<=");
-                    chars.next();
+                    self.next_char();
                 }
 
-                '!' if delims.is_empty() && chars.peek() != Some(&'=') => {
+                '!' if delims.is_empty() && self.peek() != Some(&'=') => {
                     if expression.trim().is_empty() {
                         return Err(EmptyExpression);
                     }
 
-                    conversion = match chars.next() {
+                    conversion = match self.next_char() {
                         Some('s') => ConversionFlag::Str,
                         Some('a') => ConversionFlag::Ascii,
                         Some('r') => ConversionFlag::Repr,
@@ -82,7 +91,7 @@ impl FStringParser {
                         }
                     };
 
-                    if let Some(&peek) = chars.peek() {
+                    if let Some(&peek) = self.peek() {
                         if peek != '}' && peek != ':' {
                             return Err(if expression.trim().is_empty() {
                                 EmptyExpression
@@ -101,17 +110,16 @@ impl FStringParser {
 
                 // match a python 3.8 self documenting expression
                 // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}'
-                '=' if chars.peek() != Some(&'=') && delims.is_empty() => {
+                '=' if self.peek() != Some(&'=') && delims.is_empty() => {
                     self_documenting = true;
                 }
 
                 ':' if delims.is_empty() => {
-                    let (parsed_spec, remaining_chars) = self.parse_spec(chars, nested)?;
+                    let parsed_spec = self.parse_spec(nested)?;
 
                     spec = Some(Box::new(self.expr(ExprKind::JoinedStr {
                         values: parsed_spec,
                     })));
-                    chars = remaining_chars;
                 }
                 '(' | '{' | '[' => {
                     expression.push(ch);
@@ -194,21 +202,21 @@ impl FStringParser {
                             }),
                         ]
                     };
-                    return Ok((ret, chars));
+                    return Ok(ret);
                 }
                 '"' | '\'' => {
                     expression.push(ch);
-                    let mut string_ended = false;
-                    for next in &mut chars {
-                        expression.push(next);
-                        if next == ch {
-                            string_ended = true;
-                            break;
+                    loop {
+                        match self.next_char() {
+                            Some(c) => {
+                                expression.push(c);
+                                if c == ch {
+                                    break;
+                                }
+                            }
+                            None => return Err(UnterminatedString),
                         }
                     }
-                    if !string_ended {
-                        return Err(UnterminatedString);
-                    }
                 }
                 ' ' if self_documenting => {
                     trailing_seq.push(ch);
@@ -230,14 +238,10 @@ impl FStringParser {
         })
     }
 
-    fn parse_spec<'a>(
-        &mut self,
-        mut chars: iter::Peekable<str::Chars<'a>>,
-        nested: u8,
-    ) -> Result<(Vec<Expr>, iter::Peekable<str::Chars<'a>>), FStringErrorType> {
+    fn parse_spec(&mut self, nested: u8) -> Result<Vec<Expr>, FStringErrorType> {
         let mut spec_constructor = Vec::new();
         let mut constant_piece = String::new();
-        while let Some(&next) = chars.peek() {
+        while let Some(&next) = self.peek() {
             match next {
                 '{' => {
                     if !constant_piece.is_empty() {
@@ -247,9 +251,8 @@ impl FStringParser {
                         }));
                         constant_piece.clear();
                     }
-                    let (parsed_expr, remaining_chars) = self.parse(chars, nested + 1)?;
+                    let parsed_expr = self.parse(nested + 1)?;
                     spec_constructor.extend(parsed_expr);
-                    chars = remaining_chars;
                     continue;
                 }
                 '}' => {
@@ -259,7 +262,7 @@ impl FStringParser {
                     constant_piece.push(next);
                 }
             }
-            chars.next();
+            self.next_char();
         }
         if !constant_piece.is_empty() {
             spec_constructor.push(self.expr(ExprKind::Constant {
@@ -268,14 +271,10 @@ impl FStringParser {
             }));
             constant_piece.clear();
         }
-        Ok((spec_constructor, chars))
+        Ok(spec_constructor)
     }
 
-    fn parse<'a>(
-        &mut self,
-        mut chars: iter::Peekable<str::Chars<'a>>,
-        nested: u8,
-    ) -> Result<(Vec<Expr>, iter::Peekable<str::Chars<'a>>), FStringErrorType> {
+    fn parse(&mut self, nested: u8) -> Result<Vec<Expr>, FStringErrorType> {
         if nested >= 2 {
             return Err(ExpressionNestedTooDeeply);
         }
@@ -283,14 +282,14 @@ impl FStringParser {
         let mut content = String::new();
         let mut values = vec![];
 
-        while let Some(&ch) = chars.peek() {
+        while let Some(&ch) = self.peek() {
             match ch {
                 '{' => {
-                    chars.next();
+                    self.next_char();
                     if nested == 0 {
-                        match chars.peek() {
+                        match self.peek() {
                             Some('{') => {
-                                chars.next();
+                                self.next_char();
                                 content.push('{');
                                 continue;
                             }
@@ -305,18 +304,16 @@ impl FStringParser {
                         }));
                     }
 
-                    let (parsed_values, remaining_chars) =
-                        self.parse_formatted_value(chars, nested)?;
+                    let parsed_values = self.parse_formatted_value(nested)?;
                     values.extend(parsed_values);
-                    chars = remaining_chars;
                 }
                 '}' => {
                     if nested > 0 {
                         break;
                     }
-                    chars.next();
-                    if let Some('}') = chars.peek() {
-                        chars.next();
+                    self.next_char();
+                    if let Some('}') = self.peek() {
+                        self.next_char();
                         content.push('}');
                     } else {
                         return Err(SingleRbrace);
@@ -324,7 +321,7 @@ impl FStringParser {
                 }
                 _ => {
                     content.push(ch);
-                    chars.next();
+                    self.next_char();
                 }
             }
         }
@@ -336,7 +333,7 @@ impl FStringParser {
             }))
         }
 
-        Ok((values, chars))
+        Ok(values)
     }
 }
 
@@ -352,9 +349,8 @@ pub fn parse_located_fstring(
     start: Location,
     end: Location,
 ) -> Result<Vec<Expr>, FStringError> {
-    FStringParser::new(start, end)
-        .parse(source.chars().peekable(), 0)
-        .map(|(e, _)| e)
+    FStringParser::new(source, start, end)
+        .parse(0)
         .map_err(|error| FStringError {
             error,
             location: start,
@@ -366,9 +362,7 @@ mod tests {
     use super::*;
 
     fn parse_fstring(source: &str) -> Result<Vec<Expr>, FStringErrorType> {
-        FStringParser::new(Location::default(), Location::default())
-            .parse(source.chars().peekable(), 0)
-            .map(|(e, _)| e)
+        FStringParser::new(source, Location::default(), Location::default()).parse(0)
     }
 
     #[test]

From 65739473c4fc811e768d91562602f21c61b6ec4b Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Thu, 29 Dec 2022 01:01:41 +0900
Subject: [PATCH 342/459] Rename

---
 parser/src/fstring.rs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index f92d6425..c63719e4 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -7,7 +7,7 @@ use crate::{
 use std::{iter, mem, str};
 
 struct FStringParser<'a> {
-    source: iter::Peekable<str::Chars<'a>>,
+    chars: iter::Peekable<str::Chars<'a>>,
     str_start: Location,
     str_end: Location,
 }
@@ -15,18 +15,18 @@ struct FStringParser<'a> {
 impl<'a> FStringParser<'a> {
     fn new(source: &'a str, str_start: Location, str_end: Location) -> Self {
         Self {
-            source: source.chars().peekable(),
+            chars: source.chars().peekable(),
             str_start,
             str_end,
         }
     }
 
     fn next_char(&mut self) -> Option<char> {
-        self.source.next()
+        self.chars.next()
     }
 
     fn peek(&mut self) -> Option<&char> {
-        self.source.peek()
+        self.chars.peek()
     }
 
     #[inline]

From 1d9829183985b9c5c3b360d50506a732c1e9e802 Mon Sep 17 00:00:00 2001
From: Anders Kaseorg <andersk@mit.edu>
Date: Tue, 27 Dec 2022 12:06:48 -0800
Subject: [PATCH 343/459] Allow named expression in subscript: `a[b := c]`

Signed-off-by: Anders Kaseorg <andersk@mit.edu>
---
 parser/python.lalrpop | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index a73d8352..4ec31c0c 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -1023,7 +1023,7 @@ SubscriptList: ast::Expr = {
 };
 
 Subscript: ast::Expr = {
-    Test<"all">,
+    NamedExpressionTest,
     <location:@L> <e1:Test<"all">?> ":" <e2:Test<"all">?> <e3:SliceOp?> <end_location:@R> => {
         let lower = e1.map(Box::new);
         let upper = e2.map(Box::new);

From 472d664c95375bdc13be65fb705f7ac106f47ba5 Mon Sep 17 00:00:00 2001
From: Anders Kaseorg <andersk@mit.edu>
Date: Tue, 27 Dec 2022 12:13:50 -0800
Subject: [PATCH 344/459] Allow named expression in set comprehension: `{a := b
 for c in d}`

Signed-off-by: Anders Kaseorg <andersk@mit.edu>
---
 parser/python.lalrpop | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 4ec31c0c..a49379c6 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -1185,7 +1185,7 @@ Atom<Goal>: ast::Expr = {
         custom: (),
         node: ast::ExprKind::Set { elts }
     },
-    <location:@L> "{" <elt:Test<"all">> <generators:CompFor> "}" <end_location:@R> => {
+    <location:@L> "{" <elt:NamedExpressionTest> <generators:CompFor> "}" <end_location:@R> => {
         ast::Expr {
             location,
             end_location: Some(end_location),

From 35b3bda755bdc425cb17dd9e1f164be551ad5978 Mon Sep 17 00:00:00 2001
From: Harutaka Kawamura <hkawamura0130@gmail.com>
Date: Thu, 29 Dec 2022 08:10:33 +0900
Subject: [PATCH 345/459] Update compiler/parser/src/fstring.rs

Co-authored-by: Zgarbul Andrey <zgarbul.andrey@gmail.com>
---
 parser/src/fstring.rs | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index c63719e4..c4b6e184 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -207,14 +207,12 @@ impl<'a> FStringParser<'a> {
                 '"' | '\'' => {
                     expression.push(ch);
                     loop {
-                        match self.next_char() {
-                            Some(c) => {
-                                expression.push(c);
-                                if c == ch {
-                                    break;
-                                }
-                            }
-                            None => return Err(UnterminatedString),
+                        let Some(c) = self.next_char() else {
+                            return Err(UnterminatedString);
+                        };
+                        expression.push(c);
+                        if c == ch {
+                            break;
                         }
                     }
                 }

From 8eeca31fe6d78f88eba5500c4240995755a1323c Mon Sep 17 00:00:00 2001
From: Nick Liu <nicku5011@gmail.com>
Date: Thu, 29 Dec 2022 01:12:30 +0800
Subject: [PATCH 346/459] added lex error: DuplicateArguments

---
 parser/python.lalrpop  | 55 ++++++++++++++++++++++++------------------
 parser/src/error.rs    |  4 +++
 parser/src/function.rs | 33 +++++++++++++++++++++++++
 3 files changed, 68 insertions(+), 24 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index a73d8352..70ad18f7 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -6,7 +6,7 @@
 use crate::{
     ast,
     error::{LexicalError, LexicalErrorType},
-    function::{ArgumentList, parse_args, parse_params},
+    function::{ArgumentList, parse_args, parse_params, validate_arguments},
     lexer,
     context::set_context,
     string::parse_strings,
@@ -552,16 +552,20 @@ FuncDef: ast::Stmt = {
 };
 
 Parameters: ast::Arguments = {
-    "(" <a: (ParameterList<TypedParameter>)?> ")" => {
-        a.unwrap_or_else(|| ast::Arguments {
-            posonlyargs: vec![],
-            args: vec![],
-            vararg: None,
-            kwonlyargs: vec![],
-            kw_defaults: vec![],
-            kwarg: None,
-            defaults: vec![]
-        })
+    "(" <a: (ParameterList<TypedParameter>)?> ")" =>? {
+        let args = validate_arguments(
+            a.unwrap_or_else(|| ast::Arguments {
+                posonlyargs: vec![],
+                args: vec![],
+                vararg: None,
+                kwonlyargs: vec![],
+                kw_defaults: vec![],
+                kwarg: None,
+                defaults: vec![]
+            })
+        )?;
+
+        Ok(args)
     }
 };
 
@@ -774,19 +778,22 @@ NamedExpression: ast::Expr = {
 };
 
 LambdaDef: ast::Expr = {
-    <location:@L> "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test<"all">> <end_location:@R> => {
-        let p = p.unwrap_or_else(|| {
-            ast::Arguments {
-                posonlyargs: vec![],
-                args: vec![],
-                vararg: None,
-                kwonlyargs: vec![],
-                kw_defaults: vec![],
-                kwarg: None,
-                defaults: vec![]
+    <location:@L> "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test<"all">> <end_location:@R> =>? {
+        let p = validate_arguments(
+            p.unwrap_or_else(|| {
+                ast::Arguments {
+                    posonlyargs: vec![],
+                    args: vec![],
+                    vararg: None,
+                    kwonlyargs: vec![],
+                    kw_defaults: vec![],
+                    kwarg: None,
+                    defaults: vec![]
+                }
             }
-        });
-        ast::Expr {
+        ))?;
+
+        Ok(ast::Expr {
             location,
             end_location: Some(end_location),
             custom: (),
@@ -794,7 +801,7 @@ LambdaDef: ast::Expr = {
                 args: Box::new(p),
                 body: Box::new(body)
             }
-        }
+        })
     }
 }
 
diff --git a/parser/src/error.rs b/parser/src/error.rs
index 47096a54..b9d7cbfe 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -21,6 +21,7 @@ pub enum LexicalErrorType {
     TabError,
     TabsAfterSpaces,
     DefaultArgumentError,
+    DuplicateArgumentError,
     PositionalArgumentError,
     UnpackedArgumentError,
     DuplicateKeywordArgumentError,
@@ -50,6 +51,9 @@ impl fmt::Display for LexicalErrorType {
             LexicalErrorType::DefaultArgumentError => {
                 write!(f, "non-default argument follows default argument")
             }
+            LexicalErrorType::DuplicateArgumentError => {
+                write!(f, "duplicate argument in function definition")
+            }
             LexicalErrorType::DuplicateKeywordArgumentError => {
                 write!(f, "keyword argument repeated")
             }
diff --git a/parser/src/function.rs b/parser/src/function.rs
index 46621130..adbb269e 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -10,6 +10,39 @@ pub struct ArgumentList {
 type ParameterDefs = (Vec<ast::Arg>, Vec<ast::Arg>, Vec<ast::Expr>);
 type ParameterDef = (ast::Arg, Option<ast::Expr>);
 
+pub fn validate_arguments(
+    arguments: ast::Arguments
+) -> Result<ast::Arguments, LexicalError> {
+    let mut all_args: Vec<&ast::Located<ast::ArgData>> = vec![];
+
+    all_args.extend(arguments.posonlyargs.iter());
+    all_args.extend(arguments.args.iter());
+
+    if let Some(a) = &arguments.vararg {
+        all_args.push(a);
+    }
+
+    all_args.extend(arguments.kwonlyargs.iter());
+
+    if let Some(a) = &arguments.kwarg {
+        all_args.push(a);
+    }
+
+    let mut all_arg_names =
+        FxHashSet::with_hasher(Default::default());
+    for arg in all_args {
+        let arg_name = arg.node.arg.clone();
+        if !all_arg_names.insert(arg_name) {
+            return Err(LexicalError {
+                error: LexicalErrorType::DuplicateArgumentError,
+                location: arg.location,
+            });
+        }
+    }
+
+    return Ok(arguments);
+}
+
 pub fn parse_params(
     params: (Vec<ParameterDef>, Vec<ParameterDef>),
 ) -> Result<ParameterDefs, LexicalError> {

From eccfa7dfeef91ec915ad8ead388630f6b7b48e39 Mon Sep 17 00:00:00 2001
From: Nick Liu <nicku5011@gmail.com>
Date: Thu, 29 Dec 2022 02:16:44 +0800
Subject: [PATCH 347/459] added check: named arguments must follow bare star

---
 parser/python.lalrpop | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 70ad18f7..7683b606 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -667,7 +667,7 @@ TypedParameter: ast::Arg = {
 // TODO: figure out another grammar that makes this inline no longer required.
 #[inline]
 ParameterListStarArgs<ArgType>: (Option<Box<ast::Arg>>, Vec<ast::Arg>, Vec<ast::Expr>, Option<Box<ast::Arg>>) = {
-    "*" <va:ArgType?> <kw:("," ParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> => {
+    <location:@L> "*" <va:ArgType?> <kw:("," ParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> =>? {
         // Extract keyword arguments:
         let mut kwonlyargs = Vec::new();
         let mut kw_defaults = Vec::new();
@@ -685,7 +685,14 @@ ParameterListStarArgs<ArgType>: (Option<Box<ast::Arg>>, Vec<ast::Arg>, Vec<ast::
         let kwarg = kwarg.map(|n| n.1).flatten();
         let va = va.map(Box::new);
 
-        (va, kwargs, kw_defaults, kwarg)
+        if va==None && kwargs.is_empty() && kwarg==None {
+            Err(LexicalError {
+                error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()),
+                location: location,
+            })?
+        }
+
+        Ok((va, kwargs, kw_defaults, kwarg))
     }
 };
 

From 5b40f2297c46f4eb1dc9ff2869798ea3628bd55b Mon Sep 17 00:00:00 2001
From: Nick Liu <nicku5011@gmail.com>
Date: Thu, 29 Dec 2022 21:39:38 +0800
Subject: [PATCH 348/459] add arg_name in duplicate argument error msg

---
 parser/src/error.rs    | 6 +++---
 parser/src/function.rs | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/parser/src/error.rs b/parser/src/error.rs
index b9d7cbfe..4f7b101f 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -21,7 +21,7 @@ pub enum LexicalErrorType {
     TabError,
     TabsAfterSpaces,
     DefaultArgumentError,
-    DuplicateArgumentError,
+    DuplicateArgumentError(String),
     PositionalArgumentError,
     UnpackedArgumentError,
     DuplicateKeywordArgumentError,
@@ -51,8 +51,8 @@ impl fmt::Display for LexicalErrorType {
             LexicalErrorType::DefaultArgumentError => {
                 write!(f, "non-default argument follows default argument")
             }
-            LexicalErrorType::DuplicateArgumentError => {
-                write!(f, "duplicate argument in function definition")
+            LexicalErrorType::DuplicateArgumentError(arg_name) => {
+                write!(f, "duplicate argument '{arg_name}' in function definition")
             }
             LexicalErrorType::DuplicateKeywordArgumentError => {
                 write!(f, "keyword argument repeated")
diff --git a/parser/src/function.rs b/parser/src/function.rs
index adbb269e..91f967df 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -31,16 +31,16 @@ pub fn validate_arguments(
     let mut all_arg_names =
         FxHashSet::with_hasher(Default::default());
     for arg in all_args {
-        let arg_name = arg.node.arg.clone();
+        let arg_name = &arg.node.arg;
         if !all_arg_names.insert(arg_name) {
             return Err(LexicalError {
-                error: LexicalErrorType::DuplicateArgumentError,
+                error: LexicalErrorType::DuplicateArgumentError(arg_name.to_string()),
                 location: arg.location,
             });
         }
     }
 
-    return Ok(arguments);
+    Ok(arguments)
 }
 
 pub fn parse_params(

From 1a609a9cb2e042ca22e0f2bc8674e0cf78bba849 Mon Sep 17 00:00:00 2001
From: Nick Liu <nicku5011@gmail.com>
Date: Thu, 29 Dec 2022 21:39:57 +0800
Subject: [PATCH 349/459] use is_none

---
 parser/python.lalrpop | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 7683b606..5652be1b 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -682,16 +682,16 @@ ParameterListStarArgs<ArgType>: (Option<Box<ast::Arg>>, Vec<ast::Arg>, Vec<ast::
         }
         kwargs.extend(kwonlyargs.into_iter());
 
-        let kwarg = kwarg.map(|n| n.1).flatten();
-        let va = va.map(Box::new);
-
-        if va==None && kwargs.is_empty() && kwarg==None {
+        if va.is_none() && kwargs.is_empty() && kwarg.is_none() {
             Err(LexicalError {
                 error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()),
                 location: location,
             })?
         }
 
+        let kwarg = kwarg.map(|n| n.1).flatten();
+        let va = va.map(Box::new);
+
         Ok((va, kwargs, kw_defaults, kwarg))
     }
 };

From f9b50d20b29e1b580ed632c023867e2a0f149f26 Mon Sep 17 00:00:00 2001
From: Nick Liu <nicku5011@gmail.com>
Date: Thu, 29 Dec 2022 22:42:45 +0800
Subject: [PATCH 350/459] format code

---
 parser/src/function.rs | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/parser/src/function.rs b/parser/src/function.rs
index 91f967df..1caa4598 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -10,9 +10,7 @@ pub struct ArgumentList {
 type ParameterDefs = (Vec<ast::Arg>, Vec<ast::Arg>, Vec<ast::Expr>);
 type ParameterDef = (ast::Arg, Option<ast::Expr>);
 
-pub fn validate_arguments(
-    arguments: ast::Arguments
-) -> Result<ast::Arguments, LexicalError> {
+pub fn validate_arguments(arguments: ast::Arguments) -> Result<ast::Arguments, LexicalError> {
     let mut all_args: Vec<&ast::Located<ast::ArgData>> = vec![];
 
     all_args.extend(arguments.posonlyargs.iter());
@@ -28,8 +26,7 @@ pub fn validate_arguments(
         all_args.push(a);
     }
 
-    let mut all_arg_names =
-        FxHashSet::with_hasher(Default::default());
+    let mut all_arg_names = FxHashSet::with_hasher(Default::default());
     for arg in all_args {
         let arg_name = &arg.node.arg;
         if !all_arg_names.insert(arg_name) {

From aa75f82e2dce870b6c01bd23345c7c4d4f29d146 Mon Sep 17 00:00:00 2001
From: Anil Tuncel <anil.tuncel@epfl.ch>
Date: Thu, 29 Dec 2022 22:06:21 +0100
Subject: [PATCH 351/459] arg name to be written upon duplicate kwargs error
 #4381

---
 parser/src/error.rs    | 6 +++---
 parser/src/function.rs | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/parser/src/error.rs b/parser/src/error.rs
index 4f7b101f..4a6cd6f4 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -24,7 +24,7 @@ pub enum LexicalErrorType {
     DuplicateArgumentError(String),
     PositionalArgumentError,
     UnpackedArgumentError,
-    DuplicateKeywordArgumentError,
+    DuplicateKeywordArgumentError(String),
     UnrecognizedToken { tok: char },
     FStringError(FStringErrorType),
     LineContinuationError,
@@ -54,8 +54,8 @@ impl fmt::Display for LexicalErrorType {
             LexicalErrorType::DuplicateArgumentError(arg_name) => {
                 write!(f, "duplicate argument '{arg_name}' in function definition")
             }
-            LexicalErrorType::DuplicateKeywordArgumentError => {
-                write!(f, "keyword argument repeated")
+            LexicalErrorType::DuplicateKeywordArgumentError(arg_name) => {
+                write!(f, "keyword argument '{arg_name}' repeated")
             }
             LexicalErrorType::PositionalArgumentError => {
                 write!(f, "positional argument follows keyword argument")
diff --git a/parser/src/function.rs b/parser/src/function.rs
index 1caa4598..e9d4a4f0 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -92,7 +92,7 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, Lexi
                 if let Some(keyword_name) = &name {
                     if keyword_names.contains(keyword_name) {
                         return Err(LexicalError {
-                            error: LexicalErrorType::DuplicateKeywordArgumentError,
+                            error: LexicalErrorType::DuplicateKeywordArgumentError(keyword_name.to_string()),
                             location: start,
                         });
                     }

From dc84197a23fceb147e80b5e3f659e55d241010e2 Mon Sep 17 00:00:00 2001
From: Anil Tuncel <anil.tuncel@epfl.ch>
Date: Thu, 29 Dec 2022 22:15:44 +0100
Subject: [PATCH 352/459] format using cargo fmt

---
 parser/src/function.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/parser/src/function.rs b/parser/src/function.rs
index e9d4a4f0..bb8feeab 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -92,7 +92,9 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, Lexi
                 if let Some(keyword_name) = &name {
                     if keyword_names.contains(keyword_name) {
                         return Err(LexicalError {
-                            error: LexicalErrorType::DuplicateKeywordArgumentError(keyword_name.to_string()),
+                            error: LexicalErrorType::DuplicateKeywordArgumentError(
+                                keyword_name.to_string(),
+                            ),
                             location: start,
                         });
                     }

From c4f126a8c4b68caa4325fcf29ca5e4b62f5372c8 Mon Sep 17 00:00:00 2001
From: anilbey <anil.tuncel@epfl.ch>
Date: Thu, 29 Dec 2022 22:56:34 +0100
Subject: [PATCH 353/459] Update compiler/parser/src/error.rs

Co-authored-by: fanninpm <fanninpm@miamioh.edu>
---
 parser/src/error.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/src/error.rs b/parser/src/error.rs
index 4a6cd6f4..f7a4e24a 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -55,7 +55,7 @@ impl fmt::Display for LexicalErrorType {
                 write!(f, "duplicate argument '{arg_name}' in function definition")
             }
             LexicalErrorType::DuplicateKeywordArgumentError(arg_name) => {
-                write!(f, "keyword argument '{arg_name}' repeated")
+                write!(f, "keyword argument repeated: {arg_name}")
             }
             LexicalErrorType::PositionalArgumentError => {
                 write!(f, "positional argument follows keyword argument")

From faec9372f92b430d3aa542f6530e5198cb0aa54c Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Fri, 30 Dec 2022 21:39:29 +0900
Subject: [PATCH 354/459] Fix FormattedValue location

---
 parser/python.lalrpop                         |  14 +-
 parser/src/error.rs                           |   9 +
 parser/src/fstring.rs                         |   1 +
 parser/src/lexer.rs                           | 312 ++--------
 parser/src/lib.rs                             |   2 +-
 parser/src/parser.rs                          |  23 +-
 ...er__string__tests__double_quoted_byte.snap | 297 +++++++++
 ...g__tests__escape_char_in_byte_literal.snap |  51 ++
 ...n_parser__string__tests__escape_octet.snap |  46 ++
 ...ing__tests__fstring_escaped_character.snap |  91 +++
 ...tring__tests__fstring_escaped_newline.snap |  91 +++
 ...ing__tests__fstring_line_continuation.snap |  91 +++
 ...ing__tests__fstring_unescaped_newline.snap |  91 +++
 ...tring__tests__parse_f_string_concat_3.snap |   4 +-
 ...er__string__tests__raw_byte_literal_1.snap |  45 ++
 ...er__string__tests__raw_byte_literal_2.snap |  43 ++
 ...on_parser__string__tests__raw_fstring.snap |  72 +++
 ...er__string__tests__single_quoted_byte.snap | 297 +++++++++
 ...ing__tests__triple_quoted_raw_fstring.snap |  72 +++
 parser/src/string.rs                          | 229 +++++--
 parser/src/string_parser.rs                   | 562 ++++++++++++++++++
 parser/src/token.rs                           | 107 ++--
 22 files changed, 2195 insertions(+), 355 deletions(-)
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__double_quoted_byte.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__escape_char_in_byte_literal.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__escape_octet.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__fstring_escaped_character.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__fstring_escaped_newline.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__fstring_line_continuation.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__fstring_unescaped_newline.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__raw_byte_literal_1.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__raw_byte_literal_2.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__raw_fstring.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__single_quoted_byte.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__triple_quoted_raw_fstring.snap
 create mode 100644 parser/src/string_parser.rs

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index a73d8352..90694a49 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -1339,18 +1339,11 @@ OneOrMore<T>: Vec<T> = {
 };
 
 Constant: ast::Constant = {
-    <b:bytes+> => ast::Constant::Bytes(b.into_iter().flatten().collect()),
     <value:int> => ast::Constant::Int(value),
     <value:float> => ast::Constant::Float(value),
     <s:complex> => ast::Constant::Complex { real: s.0, imag: s.1 },
 };
 
-Bytes: Vec<u8> = {
-    <s:bytes+> => {
-        s.into_iter().flatten().collect::<Vec<u8>>()
-    },
-};
-
 Identifier: String = <s:name> => s;
 
 // Hook external lexer:
@@ -1448,8 +1441,11 @@ extern {
         int => lexer::Tok::Int { value: <BigInt> },
         float => lexer::Tok::Float { value: <f64> },
         complex => lexer::Tok::Complex { real: <f64>, imag: <f64> },
-        string => lexer::Tok::String { value: <String>, kind: <StringKind> },
-        bytes => lexer::Tok::Bytes { value: <Vec<u8>> },
+        string => lexer::Tok::String {
+            value: <String>,
+            kind: <StringKind>,
+            triple_quoted: <bool>
+        },
         name => lexer::Tok::Name { name: <String> },
         "\n" => lexer::Tok::Newline,
         ";" => lexer::Tok::Semi,
diff --git a/parser/src/error.rs b/parser/src/error.rs
index 89f53668..92cb0d14 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -90,6 +90,15 @@ pub enum FStringErrorType {
     UnterminatedString,
 }
 
+impl FStringErrorType {
+    pub fn to_lexical_error(self, location: Location) -> LexicalError {
+        LexicalError {
+            error: LexicalErrorType::FStringError(self),
+            location,
+        }
+    }
+}
+
 impl fmt::Display for FStringErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
index 12b6bf8f..27a615a4 100644
--- a/parser/src/fstring.rs
+++ b/parser/src/fstring.rs
@@ -1,3 +1,4 @@
+// We no longer need this file
 use self::FStringErrorType::*;
 use crate::{
     ast::{Constant, ConversionFlag, Expr, ExprKind, Location},
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 65fb3740..db975618 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -2,8 +2,7 @@
 //!
 //! This means source code is translated into separate tokens.
 
-use super::token::StringKind;
-pub use super::token::Tok;
+pub use super::token::{StringKind, Tok};
 use crate::ast::Location;
 use crate::error::{LexicalError, LexicalErrorType};
 use num_bigint::BigInt;
@@ -217,9 +216,6 @@ where
     }
 }
 
-/// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
-const MAX_UNICODE_NAME: usize = 88;
-
 impl<T> Lexer<T>
 where
     T: Iterator<Item = char>,
@@ -274,8 +270,26 @@ where
 
             // Check if we have a string:
             if matches!(self.window[0], Some('"' | '\'')) {
+                let kind = if saw_r {
+                    if saw_b {
+                        StringKind::RawBytes
+                    } else if saw_f {
+                        StringKind::RawFString
+                    } else {
+                        StringKind::RawString
+                    }
+                } else if saw_b {
+                    StringKind::Bytes
+                } else if saw_u {
+                    StringKind::Unicode
+                } else if saw_f {
+                    StringKind::FString
+                } else {
+                    StringKind::String
+                };
+
                 return self
-                    .lex_string(saw_b, saw_r, saw_u, saw_f)
+                    .lex_string(kind)
                     .map(|(_, tok, end_pos)| (start_pos, tok, end_pos));
             }
         }
@@ -479,87 +493,7 @@ where
         }
     }
 
-    fn unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
-        let mut p: u32 = 0u32;
-        let unicode_error = LexicalError {
-            error: LexicalErrorType::UnicodeError,
-            location: self.get_pos(),
-        };
-        for i in 1..=literal_number {
-            match self.next_char() {
-                Some(c) => match c.to_digit(16) {
-                    Some(d) => p += d << ((literal_number - i) * 4),
-                    None => return Err(unicode_error),
-                },
-                None => return Err(unicode_error),
-            }
-        }
-        match p {
-            0xD800..=0xDFFF => Ok(std::char::REPLACEMENT_CHARACTER),
-            _ => std::char::from_u32(p).ok_or(unicode_error),
-        }
-    }
-
-    fn parse_octet(&mut self, first: char) -> char {
-        let mut octet_content = String::new();
-        octet_content.push(first);
-        while octet_content.len() < 3 {
-            if let Some('0'..='7') = self.window[0] {
-                octet_content.push(self.next_char().unwrap())
-            } else {
-                break;
-            }
-        }
-        let value = u32::from_str_radix(&octet_content, 8).unwrap();
-        char::from_u32(value).unwrap()
-    }
-
-    fn parse_unicode_name(&mut self) -> Result<char, LexicalError> {
-        let start_pos = self.get_pos();
-        match self.next_char() {
-            Some('{') => {}
-            _ => {
-                return Err(LexicalError {
-                    error: LexicalErrorType::StringError,
-                    location: start_pos,
-                })
-            }
-        }
-        let start_pos = self.get_pos();
-        let mut name = String::new();
-        loop {
-            match self.next_char() {
-                Some('}') => break,
-                Some(c) => name.push(c),
-                None => {
-                    return Err(LexicalError {
-                        error: LexicalErrorType::StringError,
-                        location: self.get_pos(),
-                    })
-                }
-            }
-        }
-
-        if name.len() > MAX_UNICODE_NAME {
-            return Err(LexicalError {
-                error: LexicalErrorType::UnicodeError,
-                location: self.get_pos(),
-            });
-        }
-
-        unicode_names2::character(&name).ok_or(LexicalError {
-            error: LexicalErrorType::UnicodeError,
-            location: start_pos,
-        })
-    }
-
-    fn lex_string(
-        &mut self,
-        is_bytes: bool,
-        is_raw: bool,
-        is_unicode: bool,
-        is_fstring: bool,
-    ) -> LexResult {
+    fn lex_string(&mut self, kind: StringKind) -> LexResult {
         let start_pos = self.get_pos();
         let quote_char = self.next_char().unwrap();
         let mut string_content = String::new();
@@ -577,62 +511,24 @@ where
 
         loop {
             match self.next_char() {
-                Some('\\') => {
-                    if self.window[0] == Some(quote_char) && !is_raw {
-                        string_content.push(quote_char);
-                        self.next_char();
-                    } else if is_raw {
-                        string_content.push('\\');
-                        if let Some(c) = self.next_char() {
-                            string_content.push(c)
-                        } else {
-                            return Err(LexicalError {
-                                error: LexicalErrorType::StringError,
-                                location: self.get_pos(),
-                            });
-                        }
-                    } else {
-                        match self.next_char() {
-                            Some('\\') => {
-                                string_content.push('\\');
-                            }
-                            Some('\'') => string_content.push('\''),
-                            Some('\"') => string_content.push('\"'),
-                            Some('\n') => {
-                                // Ignore Unix EOL character
-                            }
-                            Some('a') => string_content.push('\x07'),
-                            Some('b') => string_content.push('\x08'),
-                            Some('f') => string_content.push('\x0c'),
-                            Some('n') => {
-                                string_content.push('\n');
-                            }
-                            Some('r') => string_content.push('\r'),
-                            Some('t') => {
-                                string_content.push('\t');
-                            }
-                            Some('v') => string_content.push('\x0b'),
-                            Some(o @ '0'..='7') => string_content.push(self.parse_octet(o)),
-                            Some('x') => string_content.push(self.unicode_literal(2)?),
-                            Some('u') if !is_bytes => string_content.push(self.unicode_literal(4)?),
-                            Some('U') if !is_bytes => string_content.push(self.unicode_literal(8)?),
-                            Some('N') if !is_bytes => {
-                                string_content.push(self.parse_unicode_name()?)
-                            }
-                            Some(c) => {
-                                string_content.push('\\');
-                                string_content.push(c);
-                            }
-                            None => {
-                                return Err(LexicalError {
-                                    error: LexicalErrorType::StringError,
-                                    location: self.get_pos(),
-                                });
-                            }
+                Some(c) => {
+                    if c == '\\' {
+                        if let Some(next_c) = self.next_char() {
+                            string_content.push('\\');
+                            string_content.push(next_c);
+                            continue;
                         }
                     }
-                }
-                Some(c) => {
+
+                    if c == '\n' && !triple_quoted {
+                        return Err(LexicalError {
+                            error: LexicalErrorType::OtherError(
+                                "EOL while scanning string literal".to_owned(),
+                            ),
+                            location: self.get_pos(),
+                        });
+                    }
+
                     if c == quote_char {
                         if triple_quoted {
                             // Look ahead at the next two characters; if we have two more
@@ -645,19 +541,11 @@ where
                                 self.next_char();
                                 break;
                             }
-                            string_content.push(c);
                         } else {
                             break;
                         }
-                    } else {
-                        if (c == '\n' && !triple_quoted) || (is_bytes && !c.is_ascii()) {
-                            return Err(LexicalError {
-                                error: LexicalErrorType::Eof,
-                                location: self.get_pos(),
-                            });
-                        }
-                        string_content.push(c);
                     }
+                    string_content.push(c);
                 }
                 None => {
                     return Err(LexicalError {
@@ -672,25 +560,11 @@ where
             }
         }
         let end_pos = self.get_pos();
-
-        let tok = if is_bytes {
-            Tok::Bytes {
-                value: string_content.chars().map(|c| c as u8).collect(),
-            }
-        } else {
-            let kind = if is_fstring {
-                StringKind::F
-            } else if is_unicode {
-                StringKind::U
-            } else {
-                StringKind::Normal
-            };
-            Tok::String {
-                value: string_content,
-                kind,
-            }
+        let tok = Tok::String {
+            value: string_content,
+            kind,
+            triple_quoted,
         };
-
         Ok((start_pos, tok, end_pos))
     }
 
@@ -907,7 +781,7 @@ where
                 self.emit(comment);
             }
             '"' | '\'' => {
-                let string = self.lex_string(false, false, false, false)?;
+                let string = self.lex_string(StringKind::String)?;
                 self.emit(string);
             }
             '=' => {
@@ -1367,15 +1241,17 @@ mod tests {
     fn stok(s: &str) -> Tok {
         Tok::String {
             value: s.to_owned(),
-            kind: StringKind::Normal,
+            kind: StringKind::String,
+            triple_quoted: false,
         }
     }
 
-    #[test]
-    fn test_raw_string() {
-        let source = "r\"\\\\\" \"\\\\\"";
-        let tokens = lex_source(source);
-        assert_eq!(tokens, vec![stok("\\\\"), stok("\\"), Tok::Newline,]);
+    fn raw_stok(s: &str) -> Tok {
+        Tok::String {
+            value: s.to_owned(),
+            kind: StringKind::RawString,
+            triple_quoted: false,
+        }
     }
 
     #[test]
@@ -1677,13 +1553,13 @@ mod tests {
             vec![
                 stok("double"),
                 stok("single"),
-                stok("can't"),
-                stok("\\\""),
-                stok("\t\r\n"),
-                stok("\\g"),
-                stok("raw\\'"),
-                stok("Đ"),
-                stok("\u{80}\u{0}a"),
+                stok(r"can\'t"),
+                stok(r#"\\\""#),
+                stok(r"\t\r\n"),
+                stok(r"\g"),
+                raw_stok(r"raw\'"),
+                stok(r"\420"),
+                stok(r"\200\0a"),
                 Tok::Newline,
             ]
         );
@@ -1699,7 +1575,7 @@ mod tests {
                 assert_eq!(
                     tokens,
                     vec![
-                        stok("abcdef"),
+                        stok("abc\\\ndef"),
                         Tok::Newline,
                     ]
                 )
@@ -1714,78 +1590,10 @@ mod tests {
         test_string_continuation_unix_eol: UNIX_EOL,
     }
 
-    #[test]
-    fn test_single_quoted_byte() {
-        // single quote
-        let source = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##;
-        let tokens = lex_source(source);
-        let res = (0..=255).collect::<Vec<u8>>();
-        assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
-    }
-
-    #[test]
-    fn test_double_quoted_byte() {
-        // double quote
-        let source = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##;
-        let tokens = lex_source(source);
-        let res = (0..=255).collect::<Vec<u8>>();
-        assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
-    }
-
-    #[test]
-    fn test_escape_char_in_byte_literal() {
-        // backslash does not escape
-        let source = r##"b"omkmok\Xaa""##;
-        let tokens = lex_source(source);
-        let res = vec![111, 109, 107, 109, 111, 107, 92, 88, 97, 97];
-        assert_eq!(tokens, vec![Tok::Bytes { value: res }, Tok::Newline]);
-    }
-
-    #[test]
-    fn test_raw_byte_literal() {
-        let source = r"rb'\x1z'";
-        let tokens = lex_source(source);
-        assert_eq!(
-            tokens,
-            vec![
-                Tok::Bytes {
-                    value: b"\\x1z".to_vec()
-                },
-                Tok::Newline
-            ]
-        );
-        let source = r"rb'\\'";
-        let tokens = lex_source(source);
-        assert_eq!(
-            tokens,
-            vec![
-                Tok::Bytes {
-                    value: b"\\\\".to_vec()
-                },
-                Tok::Newline
-            ]
-        )
-    }
-
-    #[test]
-    fn test_escape_octet() {
-        let source = r##"b'\43a\4\1234'"##;
-        let tokens = lex_source(source);
-        assert_eq!(
-            tokens,
-            vec![
-                Tok::Bytes {
-                    value: b"#a\x04S4".to_vec()
-                },
-                Tok::Newline
-            ]
-        )
-    }
-
     #[test]
     fn test_escape_unicode_name() {
         let source = r#""\N{EN SPACE}""#;
         let tokens = lex_source(source);
-        assert_eq!(tokens, vec![stok("\u{2002}"), Tok::Newline])
+        assert_eq!(tokens, vec![stok(r"\N{EN SPACE}"), Tok::Newline])
     }
 }
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index 66e56537..d266d2a6 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -23,11 +23,11 @@ extern crate log;
 pub use rustpython_ast as ast;
 
 pub mod error;
-mod fstring;
 mod function;
 pub mod lexer;
 pub mod mode;
 pub mod parser;
+mod string_parser;
 #[rustfmt::skip]
 mod python;
 mod context;
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 705fb991..6dcbaccd 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -8,6 +8,7 @@
 use crate::lexer::{LexResult, Tok};
 pub use crate::mode::Mode;
 use crate::{ast, error::ParseError, lexer, python};
+use ast::Location;
 use itertools::Itertools;
 use std::iter;
 
@@ -65,7 +66,15 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
 ///
 /// ```
 pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> {
-    parse(source, Mode::Expression, path).map(|top| match top {
+    parse_expression_located(source, path, Location::new(1, 0))
+}
+
+pub fn parse_expression_located(
+    source: &str,
+    path: &str,
+    location: Location,
+) -> Result<ast::Expr, ParseError> {
+    parse_located(source, Mode::Expression, path, location).map(|top| match top {
         ast::Mod::Expression { body } => *body,
         _ => unreachable!(),
     })
@@ -73,7 +82,17 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
 
 // Parse a given source code
 pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, ParseError> {
-    let lxr = lexer::make_tokenizer(source);
+    parse_located(source, mode, source_path, Location::new(1, 0))
+}
+
+// Parse a given source code from a given location
+pub fn parse_located(
+    source: &str,
+    mode: Mode,
+    source_path: &str,
+    location: Location,
+) -> Result<ast::Mod, ParseError> {
+    let lxr = lexer::make_tokenizer_located(source, location);
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
     let tokenizer = iter::once(Ok(marker_token))
         .chain(lxr)
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__double_quoted_byte.snap b/parser/src/snapshots/rustpython_parser__string__tests__double_quoted_byte.snap
new file mode 100644
index 00000000..0d8c8c9a
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__double_quoted_byte.snap
@@ -0,0 +1,297 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 738,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 738,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Bytes(
+                        [
+                            0,
+                            1,
+                            2,
+                            3,
+                            4,
+                            5,
+                            6,
+                            7,
+                            8,
+                            9,
+                            10,
+                            11,
+                            12,
+                            13,
+                            14,
+                            15,
+                            16,
+                            17,
+                            18,
+                            19,
+                            20,
+                            21,
+                            22,
+                            23,
+                            24,
+                            25,
+                            26,
+                            27,
+                            28,
+                            29,
+                            30,
+                            31,
+                            32,
+                            33,
+                            34,
+                            35,
+                            36,
+                            37,
+                            38,
+                            39,
+                            40,
+                            41,
+                            42,
+                            43,
+                            44,
+                            45,
+                            46,
+                            47,
+                            48,
+                            49,
+                            50,
+                            51,
+                            52,
+                            53,
+                            54,
+                            55,
+                            56,
+                            57,
+                            58,
+                            59,
+                            60,
+                            61,
+                            62,
+                            63,
+                            64,
+                            65,
+                            66,
+                            67,
+                            68,
+                            69,
+                            70,
+                            71,
+                            72,
+                            73,
+                            74,
+                            75,
+                            76,
+                            77,
+                            78,
+                            79,
+                            80,
+                            81,
+                            82,
+                            83,
+                            84,
+                            85,
+                            86,
+                            87,
+                            88,
+                            89,
+                            90,
+                            91,
+                            92,
+                            93,
+                            94,
+                            95,
+                            96,
+                            97,
+                            98,
+                            99,
+                            100,
+                            101,
+                            102,
+                            103,
+                            104,
+                            105,
+                            106,
+                            107,
+                            108,
+                            109,
+                            110,
+                            111,
+                            112,
+                            113,
+                            114,
+                            115,
+                            116,
+                            117,
+                            118,
+                            119,
+                            120,
+                            121,
+                            122,
+                            123,
+                            124,
+                            125,
+                            126,
+                            127,
+                            128,
+                            129,
+                            130,
+                            131,
+                            132,
+                            133,
+                            134,
+                            135,
+                            136,
+                            137,
+                            138,
+                            139,
+                            140,
+                            141,
+                            142,
+                            143,
+                            144,
+                            145,
+                            146,
+                            147,
+                            148,
+                            149,
+                            150,
+                            151,
+                            152,
+                            153,
+                            154,
+                            155,
+                            156,
+                            157,
+                            158,
+                            159,
+                            160,
+                            161,
+                            162,
+                            163,
+                            164,
+                            165,
+                            166,
+                            167,
+                            168,
+                            169,
+                            170,
+                            171,
+                            172,
+                            173,
+                            174,
+                            175,
+                            176,
+                            177,
+                            178,
+                            179,
+                            180,
+                            181,
+                            182,
+                            183,
+                            184,
+                            185,
+                            186,
+                            187,
+                            188,
+                            189,
+                            190,
+                            191,
+                            192,
+                            193,
+                            194,
+                            195,
+                            196,
+                            197,
+                            198,
+                            199,
+                            200,
+                            201,
+                            202,
+                            203,
+                            204,
+                            205,
+                            206,
+                            207,
+                            208,
+                            209,
+                            210,
+                            211,
+                            212,
+                            213,
+                            214,
+                            215,
+                            216,
+                            217,
+                            218,
+                            219,
+                            220,
+                            221,
+                            222,
+                            223,
+                            224,
+                            225,
+                            226,
+                            227,
+                            228,
+                            229,
+                            230,
+                            231,
+                            232,
+                            233,
+                            234,
+                            235,
+                            236,
+                            237,
+                            238,
+                            239,
+                            240,
+                            241,
+                            242,
+                            243,
+                            244,
+                            245,
+                            246,
+                            247,
+                            248,
+                            249,
+                            250,
+                            251,
+                            252,
+                            253,
+                            254,
+                            255,
+                        ],
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__escape_char_in_byte_literal.snap b/parser/src/snapshots/rustpython_parser__string__tests__escape_char_in_byte_literal.snap
new file mode 100644
index 00000000..98fc3c79
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__escape_char_in_byte_literal.snap
@@ -0,0 +1,51 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 13,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 13,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Bytes(
+                        [
+                            111,
+                            109,
+                            107,
+                            109,
+                            111,
+                            107,
+                            92,
+                            88,
+                            97,
+                            97,
+                        ],
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__escape_octet.snap b/parser/src/snapshots/rustpython_parser__string__tests__escape_octet.snap
new file mode 100644
index 00000000..677e3f9f
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__escape_octet.snap
@@ -0,0 +1,46 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 14,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 14,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Bytes(
+                        [
+                            35,
+                            97,
+                            4,
+                            83,
+                            52,
+                        ],
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__fstring_escaped_character.snap b/parser/src/snapshots/rustpython_parser__string__tests__fstring_escaped_character.snap
new file mode 100644
index 00000000..7ab1247b
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__fstring_escaped_character.snap
@@ -0,0 +1,91 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 8,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 8,
+                    },
+                ),
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 0,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 8,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "\\",
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 0,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 8,
+                                },
+                            ),
+                            custom: (),
+                            node: FormattedValue {
+                                value: Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 5,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 6,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "x",
+                                        ctx: Load,
+                                    },
+                                },
+                                conversion: 0,
+                                format_spec: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__fstring_escaped_newline.snap b/parser/src/snapshots/rustpython_parser__string__tests__fstring_escaped_newline.snap
new file mode 100644
index 00000000..83598459
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__fstring_escaped_newline.snap
@@ -0,0 +1,91 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 8,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 8,
+                    },
+                ),
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 0,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 8,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "\n",
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 0,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 8,
+                                },
+                            ),
+                            custom: (),
+                            node: FormattedValue {
+                                value: Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 5,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 6,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "x",
+                                        ctx: Load,
+                                    },
+                                },
+                                conversion: 0,
+                                format_spec: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__fstring_line_continuation.snap b/parser/src/snapshots/rustpython_parser__string__tests__fstring_line_continuation.snap
new file mode 100644
index 00000000..2f6167d4
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__fstring_line_continuation.snap
@@ -0,0 +1,91 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 2,
+                column: 4,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 2,
+                        column: 4,
+                    },
+                ),
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 0,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 4,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "\\\n",
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 0,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 4,
+                                },
+                            ),
+                            custom: (),
+                            node: FormattedValue {
+                                value: Located {
+                                    location: Location {
+                                        row: 2,
+                                        column: 1,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 2,
+                                            column: 2,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "x",
+                                        ctx: Load,
+                                    },
+                                },
+                                conversion: 0,
+                                format_spec: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__fstring_unescaped_newline.snap b/parser/src/snapshots/rustpython_parser__string__tests__fstring_unescaped_newline.snap
new file mode 100644
index 00000000..b44c3299
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__fstring_unescaped_newline.snap
@@ -0,0 +1,91 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 2,
+                column: 6,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 2,
+                        column: 6,
+                    },
+                ),
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 0,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 6,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "\n",
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 0,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 6,
+                                },
+                            ),
+                            custom: (),
+                            node: FormattedValue {
+                                value: Located {
+                                    location: Location {
+                                        row: 2,
+                                        column: 1,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 2,
+                                            column: 2,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "x",
+                                        ctx: Load,
+                                    },
+                                },
+                                conversion: 0,
+                                format_spec: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
index 87b92127..0fdc8e82 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap
@@ -65,12 +65,12 @@ expression: parse_ast
                                 value: Located {
                                     location: Location {
                                         row: 1,
-                                        column: 1,
+                                        column: 17,
                                     },
                                     end_location: Some(
                                         Location {
                                             row: 1,
-                                            column: 4,
+                                            column: 20,
                                         },
                                     ),
                                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__raw_byte_literal_1.snap b/parser/src/snapshots/rustpython_parser__string__tests__raw_byte_literal_1.snap
new file mode 100644
index 00000000..14daedd9
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__raw_byte_literal_1.snap
@@ -0,0 +1,45 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 8,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 8,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Bytes(
+                        [
+                            92,
+                            120,
+                            49,
+                            122,
+                        ],
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__raw_byte_literal_2.snap b/parser/src/snapshots/rustpython_parser__string__tests__raw_byte_literal_2.snap
new file mode 100644
index 00000000..d34d8c88
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__raw_byte_literal_2.snap
@@ -0,0 +1,43 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 6,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 6,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Bytes(
+                        [
+                            92,
+                            92,
+                        ],
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__raw_fstring.snap b/parser/src/snapshots/rustpython_parser__string__tests__raw_fstring.snap
new file mode 100644
index 00000000..e53b8610
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__raw_fstring.snap
@@ -0,0 +1,72 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 7,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 0,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 7,
+                                },
+                            ),
+                            custom: (),
+                            node: FormattedValue {
+                                value: Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 4,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 5,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "x",
+                                        ctx: Load,
+                                    },
+                                },
+                                conversion: 0,
+                                format_spec: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__single_quoted_byte.snap b/parser/src/snapshots/rustpython_parser__string__tests__single_quoted_byte.snap
new file mode 100644
index 00000000..0d8c8c9a
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__single_quoted_byte.snap
@@ -0,0 +1,297 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 738,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 738,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Bytes(
+                        [
+                            0,
+                            1,
+                            2,
+                            3,
+                            4,
+                            5,
+                            6,
+                            7,
+                            8,
+                            9,
+                            10,
+                            11,
+                            12,
+                            13,
+                            14,
+                            15,
+                            16,
+                            17,
+                            18,
+                            19,
+                            20,
+                            21,
+                            22,
+                            23,
+                            24,
+                            25,
+                            26,
+                            27,
+                            28,
+                            29,
+                            30,
+                            31,
+                            32,
+                            33,
+                            34,
+                            35,
+                            36,
+                            37,
+                            38,
+                            39,
+                            40,
+                            41,
+                            42,
+                            43,
+                            44,
+                            45,
+                            46,
+                            47,
+                            48,
+                            49,
+                            50,
+                            51,
+                            52,
+                            53,
+                            54,
+                            55,
+                            56,
+                            57,
+                            58,
+                            59,
+                            60,
+                            61,
+                            62,
+                            63,
+                            64,
+                            65,
+                            66,
+                            67,
+                            68,
+                            69,
+                            70,
+                            71,
+                            72,
+                            73,
+                            74,
+                            75,
+                            76,
+                            77,
+                            78,
+                            79,
+                            80,
+                            81,
+                            82,
+                            83,
+                            84,
+                            85,
+                            86,
+                            87,
+                            88,
+                            89,
+                            90,
+                            91,
+                            92,
+                            93,
+                            94,
+                            95,
+                            96,
+                            97,
+                            98,
+                            99,
+                            100,
+                            101,
+                            102,
+                            103,
+                            104,
+                            105,
+                            106,
+                            107,
+                            108,
+                            109,
+                            110,
+                            111,
+                            112,
+                            113,
+                            114,
+                            115,
+                            116,
+                            117,
+                            118,
+                            119,
+                            120,
+                            121,
+                            122,
+                            123,
+                            124,
+                            125,
+                            126,
+                            127,
+                            128,
+                            129,
+                            130,
+                            131,
+                            132,
+                            133,
+                            134,
+                            135,
+                            136,
+                            137,
+                            138,
+                            139,
+                            140,
+                            141,
+                            142,
+                            143,
+                            144,
+                            145,
+                            146,
+                            147,
+                            148,
+                            149,
+                            150,
+                            151,
+                            152,
+                            153,
+                            154,
+                            155,
+                            156,
+                            157,
+                            158,
+                            159,
+                            160,
+                            161,
+                            162,
+                            163,
+                            164,
+                            165,
+                            166,
+                            167,
+                            168,
+                            169,
+                            170,
+                            171,
+                            172,
+                            173,
+                            174,
+                            175,
+                            176,
+                            177,
+                            178,
+                            179,
+                            180,
+                            181,
+                            182,
+                            183,
+                            184,
+                            185,
+                            186,
+                            187,
+                            188,
+                            189,
+                            190,
+                            191,
+                            192,
+                            193,
+                            194,
+                            195,
+                            196,
+                            197,
+                            198,
+                            199,
+                            200,
+                            201,
+                            202,
+                            203,
+                            204,
+                            205,
+                            206,
+                            207,
+                            208,
+                            209,
+                            210,
+                            211,
+                            212,
+                            213,
+                            214,
+                            215,
+                            216,
+                            217,
+                            218,
+                            219,
+                            220,
+                            221,
+                            222,
+                            223,
+                            224,
+                            225,
+                            226,
+                            227,
+                            228,
+                            229,
+                            230,
+                            231,
+                            232,
+                            233,
+                            234,
+                            235,
+                            236,
+                            237,
+                            238,
+                            239,
+                            240,
+                            241,
+                            242,
+                            243,
+                            244,
+                            245,
+                            246,
+                            247,
+                            248,
+                            249,
+                            250,
+                            251,
+                            252,
+                            253,
+                            254,
+                            255,
+                        ],
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__triple_quoted_raw_fstring.snap b/parser/src/snapshots/rustpython_parser__string__tests__triple_quoted_raw_fstring.snap
new file mode 100644
index 00000000..19975d53
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__triple_quoted_raw_fstring.snap
@@ -0,0 +1,72 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 11,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 11,
+                    },
+                ),
+                custom: (),
+                node: JoinedStr {
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 0,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 11,
+                                },
+                            ),
+                            custom: (),
+                            node: FormattedValue {
+                                value: Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 6,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 7,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "x",
+                                        ctx: Load,
+                                    },
+                                },
+                                conversion: 0,
+                                format_spec: None,
+                            },
+                        },
+                    ],
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/string.rs b/parser/src/string.rs
index b80c6298..817d0e88 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -1,35 +1,79 @@
 use crate::{
     ast::{Constant, Expr, ExprKind, Location},
     error::{LexicalError, LexicalErrorType},
-    fstring::parse_located_fstring,
+    string_parser::parse_string,
     token::StringKind,
 };
 use itertools::Itertools;
 
 pub fn parse_strings(
-    values: Vec<(Location, (String, StringKind), Location)>,
+    values: Vec<(Location, (String, StringKind, bool), Location)>,
 ) -> Result<Expr, LexicalError> {
     // Preserve the initial location and kind.
     let initial_start = values[0].0;
     let last_end = values.last().unwrap().2;
-    let initial_kind = (values[0].1 .1 == StringKind::U).then(|| "u".to_owned());
+    let initial_kind = (values[0].1 .1 == StringKind::Unicode).then(|| "u".to_owned());
+    let has_fstring = values.iter().any(|(_, (_, kind, ..), _)| kind.is_fstring());
+    let num_bytes = values
+        .iter()
+        .filter(|(_, (_, kind, ..), _)| kind.is_bytes())
+        .count();
+    let has_bytes = num_bytes > 0;
 
-    // Optimization: fast-track the common case of a single string.
-    if matches!(&*values, [(_, (_, StringKind::Normal | StringKind::U), _)]) {
-        let value = values.into_iter().last().unwrap().1 .0;
+    if has_bytes && num_bytes < values.len() {
+        return Err(LexicalError {
+            error: LexicalErrorType::OtherError(
+                "cannot mix bytes and nonbytes literals".to_owned(),
+            ),
+            location: initial_start,
+        });
+    }
+
+    if has_bytes {
+        let mut content: Vec<u8> = vec![];
+        for (start, (source, kind, triple_quoted), end) in values {
+            for value in parse_string(&source, kind, triple_quoted, start, end)? {
+                match value.node {
+                    ExprKind::Constant {
+                        value: Constant::Bytes(value),
+                        ..
+                    } => content.extend(value),
+                    _ => unreachable!("Unexpected non-bytes expression."),
+                }
+            }
+        }
         return Ok(Expr::new(
             initial_start,
             last_end,
             ExprKind::Constant {
-                value: Constant::Str(value),
-                kind: initial_kind,
+                value: Constant::Bytes(content),
+                kind: None,
             },
         ));
     }
 
-    // Determine whether the list of values contains any f-strings. (If not, we can return a
-    // single Constant at the end, rather than a JoinedStr.)
-    let mut has_fstring = false;
+    if !has_fstring {
+        let mut content: Vec<String> = vec![];
+        for (start, (source, kind, triple_quoted), end) in values {
+            for value in parse_string(&source, kind, triple_quoted, start, end)? {
+                match value.node {
+                    ExprKind::Constant {
+                        value: Constant::Str(value),
+                        ..
+                    } => content.push(value),
+                    _ => unreachable!("Unexpected non-string expression."),
+                }
+            }
+        }
+        return Ok(Expr::new(
+            initial_start,
+            last_end,
+            ExprKind::Constant {
+                value: Constant::Str(content.join("")),
+                kind: initial_kind,
+            },
+        ));
+    }
 
     // De-duplicate adjacent constants.
     let mut deduped: Vec<Expr> = vec![];
@@ -46,34 +90,20 @@ pub fn parse_strings(
         )
     };
 
-    for (start, (string, string_kind), end) in values {
-        match string_kind {
-            StringKind::Normal | StringKind::U => current.push(string),
-            StringKind::F => {
-                has_fstring = true;
-                for value in
-                    parse_located_fstring(&string, start, end).map_err(|e| LexicalError {
-                        location: start,
-                        error: LexicalErrorType::FStringError(e.error),
-                    })?
-                {
-                    match value.node {
-                        ExprKind::FormattedValue { .. } => {
-                            if !current.is_empty() {
-                                deduped.push(take_current(&mut current));
-                            }
-                            deduped.push(value)
-                        }
-                        ExprKind::Constant { value, .. } => {
-                            if let Constant::Str(value) = value {
-                                current.push(value);
-                            } else {
-                                unreachable!("Unexpected non-string constant.");
-                            }
-                        }
-                        _ => unreachable!("Unexpected non-string expression."),
+    for (start, (source, kind, triple_quoted), end) in values {
+        for value in parse_string(&source, kind, triple_quoted, start, end)? {
+            match value.node {
+                ExprKind::FormattedValue { .. } => {
+                    if !current.is_empty() {
+                        deduped.push(take_current(&mut current));
                     }
+                    deduped.push(value)
                 }
+                ExprKind::Constant {
+                    value: Constant::Str(value),
+                    ..
+                } => current.push(value),
+                _ => unreachable!("Unexpected non-string expression."),
             }
         }
     }
@@ -101,64 +131,153 @@ mod tests {
 
     #[test]
     fn test_parse_string_concat() {
-        let source = String::from("'Hello ' 'world'");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "'Hello ' 'world'";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_string_concat_1() {
-        let source = String::from("'Hello ' u'world'");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "'Hello ' u'world'";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_string_concat_2() {
-        let source = String::from("u'Hello ' 'world'");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "u'Hello ' 'world'";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string_concat_1() {
-        let source = String::from("'Hello ' f'world'");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "'Hello ' f'world'";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string_concat_2() {
-        let source = String::from("'Hello ' f'world'");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "'Hello ' f'world'";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string_concat_3() {
-        let source = String::from("'Hello ' f'world{\"!\"}'");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "'Hello ' f'world{\"!\"}'";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_f_string_concat_1() {
-        let source = String::from("u'Hello ' f'world'");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "u'Hello ' f'world'";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_f_string_concat_2() {
-        let source = String::from("u'Hello ' f'world' '!'");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "u'Hello ' f'world' '!'";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_string_triple_quotes_with_kind() {
-        let source = String::from("u'''Hello, world!'''");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "u'''Hello, world!'''";
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_single_quoted_byte() {
+        // single quote
+        let source = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##;
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_double_quoted_byte() {
+        // double quote
+        let source = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##;
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_escape_char_in_byte_literal() {
+        // backslash does not escape
+        let source = r##"b"omkmok\Xaa""##;
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_raw_byte_literal_1() {
+        let source = r"rb'\x1z'";
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_raw_byte_literal_2() {
+        let source = r"rb'\\'";
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_escape_octet() {
+        let source = r##"b'\43a\4\1234'"##;
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_fstring_escaped_newline() {
+        let source = r#"f"\n{x}""#;
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_fstring_unescaped_newline() {
+        let source = r#"f"""
+{x}""""#;
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_fstring_escaped_character() {
+        let source = r#"f"\\{x}""#;
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_raw_fstring() {
+        let source = r#"rf"{x}""#;
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_triple_quoted_raw_fstring() {
+        let source = r#"rf"""{x}""""#;
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_fstring_line_continuation() {
+        let source = r#"rf"\
+{x}""#;
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 }
diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
new file mode 100644
index 00000000..610951ac
--- /dev/null
+++ b/parser/src/string_parser.rs
@@ -0,0 +1,562 @@
+use self::FStringErrorType::*;
+use crate::{
+    ast::{Constant, ConversionFlag, Expr, ExprKind, Location},
+    error::{FStringErrorType, LexicalError, LexicalErrorType, ParseError},
+    parser::parse_expression_located,
+    token::StringKind,
+};
+use std::{iter, mem, str};
+
+/// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
+pub const MAX_UNICODE_NAME: usize = 88;
+
+pub struct StringParser<'a> {
+    chars: iter::Peekable<str::Chars<'a>>,
+    kind: StringKind,
+    str_start: Location,
+    str_end: Location,
+    location: Location,
+}
+
+impl<'a> StringParser<'a> {
+    pub fn new(
+        source: &'a str,
+        kind: StringKind,
+        triple_quoted: bool,
+        str_start: Location,
+        str_end: Location,
+    ) -> Self {
+        let offset = kind.to_string().len() + if triple_quoted { 3 } else { 1 };
+        Self {
+            chars: source.chars().peekable(),
+            kind,
+            str_start,
+            str_end,
+            location: Location::new(str_start.row(), str_start.column() + offset),
+        }
+    }
+
+    fn next_char(&mut self) -> Option<char> {
+        let Some(c) = self.chars.next() else {
+            return None
+        };
+        if c == '\n' {
+            self.location.newline();
+        } else {
+            self.location.go_right();
+        }
+        Some(c)
+    }
+
+    fn peek(&mut self) -> Option<&char> {
+        self.chars.peek()
+    }
+
+    fn get_pos(&self) -> Location {
+        self.location
+    }
+
+    #[inline]
+    fn expr(&self, node: ExprKind) -> Expr {
+        Expr::new(self.str_start, self.str_end, node)
+    }
+
+    fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
+        let mut p: u32 = 0u32;
+        let unicode_error = LexicalError {
+            error: LexicalErrorType::UnicodeError,
+            location: self.get_pos(),
+        };
+        for i in 1..=literal_number {
+            match self.next_char() {
+                Some(c) => match c.to_digit(16) {
+                    Some(d) => p += d << ((literal_number - i) * 4),
+                    None => return Err(unicode_error),
+                },
+                None => return Err(unicode_error),
+            }
+        }
+        match p {
+            0xD800..=0xDFFF => Ok(std::char::REPLACEMENT_CHARACTER),
+            _ => std::char::from_u32(p).ok_or(unicode_error),
+        }
+    }
+
+    fn parse_octet(&mut self, first: char) -> char {
+        let mut octet_content = String::new();
+        octet_content.push(first);
+        while octet_content.len() < 3 {
+            if let Some('0'..='7') = self.peek() {
+                octet_content.push(self.next_char().unwrap())
+            } else {
+                break;
+            }
+        }
+        let value = u32::from_str_radix(&octet_content, 8).unwrap();
+        char::from_u32(value).unwrap()
+    }
+
+    fn parse_unicode_name(&mut self) -> Result<char, LexicalError> {
+        let start_pos = self.get_pos();
+        match self.next_char() {
+            Some('{') => {}
+            _ => {
+                return Err(LexicalError {
+                    error: LexicalErrorType::StringError,
+                    location: start_pos,
+                })
+            }
+        }
+        let start_pos = self.get_pos();
+        let mut name = String::new();
+        loop {
+            match self.next_char() {
+                Some('}') => break,
+                Some(c) => name.push(c),
+                None => {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::StringError,
+                        location: self.get_pos(),
+                    })
+                }
+            }
+        }
+
+        if name.len() > MAX_UNICODE_NAME {
+            return Err(LexicalError {
+                error: LexicalErrorType::UnicodeError,
+                location: self.get_pos(),
+            });
+        }
+
+        unicode_names2::character(&name).ok_or(LexicalError {
+            error: LexicalErrorType::UnicodeError,
+            location: start_pos,
+        })
+    }
+
+    fn parse_escaped_char(&mut self) -> Result<String, LexicalError> {
+        match self.next_char() {
+            Some(c) => Ok(match c {
+                '\\' => '\\'.to_string(),
+                '\'' => '\''.to_string(),
+                '\"' => '"'.to_string(),
+                '\n' => "".to_string(),
+                'a' => '\x07'.to_string(),
+                'b' => '\x08'.to_string(),
+                'f' => '\x0c'.to_string(),
+                'n' => '\n'.to_string(),
+                'r' => '\r'.to_string(),
+                't' => '\t'.to_string(),
+                'v' => '\x0b'.to_string(),
+                o @ '0'..='7' => self.parse_octet(o).to_string(),
+                'x' => self.parse_unicode_literal(2)?.to_string(),
+                'u' if !self.kind.is_bytes() => self.parse_unicode_literal(4)?.to_string(),
+                'U' if !self.kind.is_bytes() => self.parse_unicode_literal(8)?.to_string(),
+                'N' if !self.kind.is_bytes() => self.parse_unicode_name()?.to_string(),
+                c => {
+                    if self.kind.is_bytes() && !c.is_ascii() {
+                        return Err(LexicalError {
+                            error: LexicalErrorType::OtherError(
+                                "bytes can only contain ASCII literal characters".to_owned(),
+                            ),
+                            location: self.get_pos(),
+                        });
+                    }
+                    format!("\\{c}")
+                }
+            }),
+            None => Err(LexicalError {
+                error: LexicalErrorType::StringError,
+                location: self.get_pos(),
+            }),
+        }
+    }
+
+    fn parse_formatted_value(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
+        let mut expression = String::new();
+        let mut spec = None;
+        let mut delims = Vec::new();
+        let mut conversion = ConversionFlag::None;
+        let mut self_documenting = false;
+        let mut trailing_seq = String::new();
+        let location = self.get_pos();
+
+        while let Some(ch) = self.next_char() {
+            match ch {
+                // can be integrated better with the remaining code, but as a starting point ok
+                // in general I would do here a tokenizing of the fstrings to omit this peeking.
+                '!' if self.peek() == Some(&'=') => {
+                    expression.push_str("!=");
+                    self.next_char();
+                }
+
+                '=' if self.peek() == Some(&'=') => {
+                    expression.push_str("==");
+                    self.next_char();
+                }
+
+                '>' if self.peek() == Some(&'=') => {
+                    expression.push_str(">=");
+                    self.next_char();
+                }
+
+                '<' if self.peek() == Some(&'=') => {
+                    expression.push_str("<=");
+                    self.next_char();
+                }
+
+                '!' if delims.is_empty() && self.peek() != Some(&'=') => {
+                    if expression.trim().is_empty() {
+                        return Err(EmptyExpression.to_lexical_error(self.get_pos()));
+                    }
+
+                    conversion = match self.next_char() {
+                        Some('s') => ConversionFlag::Str,
+                        Some('a') => ConversionFlag::Ascii,
+                        Some('r') => ConversionFlag::Repr,
+                        Some(_) => {
+                            return Err(if expression.trim().is_empty() {
+                                EmptyExpression.to_lexical_error(self.get_pos())
+                            } else {
+                                InvalidConversionFlag.to_lexical_error(self.get_pos())
+                            });
+                        }
+                        None => {
+                            return Err(if expression.trim().is_empty() {
+                                EmptyExpression.to_lexical_error(self.get_pos())
+                            } else {
+                                UnclosedLbrace.to_lexical_error(self.get_pos())
+                            });
+                        }
+                    };
+
+                    if let Some(&peek) = self.peek() {
+                        if peek != '}' && peek != ':' {
+                            return Err(if expression.trim().is_empty() {
+                                EmptyExpression.to_lexical_error(self.get_pos())
+                            } else {
+                                UnclosedLbrace.to_lexical_error(self.get_pos())
+                            });
+                        }
+                    } else {
+                        return Err(if expression.trim().is_empty() {
+                            EmptyExpression.to_lexical_error(self.get_pos())
+                        } else {
+                            UnclosedLbrace.to_lexical_error(self.get_pos())
+                        });
+                    }
+                }
+
+                // match a python 3.8 self documenting expression
+                // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}'
+                '=' if self.peek() != Some(&'=') && delims.is_empty() => {
+                    self_documenting = true;
+                }
+
+                ':' if delims.is_empty() => {
+                    let parsed_spec = self.parse_spec(nested)?;
+
+                    spec = Some(Box::new(self.expr(ExprKind::JoinedStr {
+                        values: parsed_spec,
+                    })));
+                }
+                '(' | '{' | '[' => {
+                    expression.push(ch);
+                    delims.push(ch);
+                }
+                ')' => {
+                    let last_delim = delims.pop();
+                    match last_delim {
+                        Some('(') => {
+                            expression.push(ch);
+                        }
+                        Some(c) => {
+                            return Err(
+                                MismatchedDelimiter(c, ')').to_lexical_error(self.get_pos())
+                            );
+                        }
+                        None => {
+                            return Err(Unmatched(')').to_lexical_error(self.get_pos()));
+                        }
+                    }
+                }
+                ']' => {
+                    let last_delim = delims.pop();
+                    match last_delim {
+                        Some('[') => {
+                            expression.push(ch);
+                        }
+                        Some(c) => {
+                            return Err(
+                                MismatchedDelimiter(c, ']').to_lexical_error(self.get_pos())
+                            );
+                        }
+                        None => {
+                            return Err(Unmatched(']').to_lexical_error(self.get_pos()));
+                        }
+                    }
+                }
+                '}' if !delims.is_empty() => {
+                    let last_delim = delims.pop();
+                    match last_delim {
+                        Some('{') => {
+                            expression.push(ch);
+                        }
+                        Some(c) => {
+                            return Err(MismatchedDelimiter(c, '}').to_lexical_error(self.get_pos()))
+                        }
+                        None => {}
+                    }
+                }
+                '}' => {
+                    if expression.trim().is_empty() {
+                        return Err(EmptyExpression.to_lexical_error(self.get_pos()));
+                    }
+
+                    let ret = if !self_documenting {
+                        vec![self.expr(ExprKind::FormattedValue {
+                            value: Box::new(parse_fstring_expr(&expression, location).map_err(
+                                |e| {
+                                    InvalidExpression(Box::new(e.error))
+                                        .to_lexical_error(self.get_pos())
+                                },
+                            )?),
+                            conversion: conversion as _,
+                            format_spec: spec,
+                        })]
+                    } else {
+                        vec![
+                            self.expr(ExprKind::Constant {
+                                value: Constant::Str(expression.to_owned() + "="),
+                                kind: None,
+                            }),
+                            self.expr(ExprKind::Constant {
+                                value: trailing_seq.into(),
+                                kind: None,
+                            }),
+                            self.expr(ExprKind::FormattedValue {
+                                value: Box::new(
+                                    parse_fstring_expr(&expression, location).map_err(|e| {
+                                        InvalidExpression(Box::new(e.error))
+                                            .to_lexical_error(self.get_pos())
+                                    })?,
+                                ),
+                                conversion: (if conversion == ConversionFlag::None && spec.is_none()
+                                {
+                                    ConversionFlag::Repr
+                                } else {
+                                    conversion
+                                }) as _,
+                                format_spec: spec,
+                            }),
+                        ]
+                    };
+                    return Ok(ret);
+                }
+                '"' | '\'' => {
+                    expression.push(ch);
+                    loop {
+                        let Some(c) = self.next_char() else {
+                            return Err(UnterminatedString.to_lexical_error(self.get_pos()));
+                        };
+                        expression.push(c);
+                        if c == ch {
+                            break;
+                        }
+                    }
+                }
+                ' ' if self_documenting => {
+                    trailing_seq.push(ch);
+                }
+                '\\' => return Err(ExpressionCannotInclude('\\').to_lexical_error(self.get_pos())),
+                _ => {
+                    if self_documenting {
+                        return Err(UnclosedLbrace.to_lexical_error(self.get_pos()));
+                    }
+
+                    expression.push(ch);
+                }
+            }
+        }
+        Err(if expression.trim().is_empty() {
+            EmptyExpression.to_lexical_error(self.get_pos())
+        } else {
+            UnclosedLbrace.to_lexical_error(self.get_pos())
+        })
+    }
+
+    fn parse_spec(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
+        let mut spec_constructor = Vec::new();
+        let mut constant_piece = String::new();
+        while let Some(&next) = self.peek() {
+            match next {
+                '{' => {
+                    if !constant_piece.is_empty() {
+                        spec_constructor.push(self.expr(ExprKind::Constant {
+                            value: constant_piece.to_owned().into(),
+                            kind: None,
+                        }));
+                        constant_piece.clear();
+                    }
+                    let parsed_expr = self.parse_fstring(nested + 1)?;
+                    spec_constructor.extend(parsed_expr);
+                    continue;
+                }
+                '}' => {
+                    break;
+                }
+                _ => {
+                    constant_piece.push(next);
+                }
+            }
+            self.next_char();
+        }
+        if !constant_piece.is_empty() {
+            spec_constructor.push(self.expr(ExprKind::Constant {
+                value: constant_piece.to_owned().into(),
+                kind: None,
+            }));
+            constant_piece.clear();
+        }
+        Ok(spec_constructor)
+    }
+
+    fn parse_fstring(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
+        if nested >= 2 {
+            return Err(ExpressionNestedTooDeeply.to_lexical_error(self.get_pos()));
+        }
+
+        let mut content = String::new();
+        let mut values = vec![];
+
+        while let Some(&ch) = self.peek() {
+            match ch {
+                '{' => {
+                    self.next_char();
+                    if nested == 0 {
+                        match self.peek() {
+                            Some('{') => {
+                                self.next_char();
+                                content.push('{');
+                                continue;
+                            }
+                            None => return Err(UnclosedLbrace.to_lexical_error(self.get_pos())),
+                            _ => {}
+                        }
+                    }
+                    if !content.is_empty() {
+                        values.push(self.expr(ExprKind::Constant {
+                            value: mem::take(&mut content).into(),
+                            kind: None,
+                        }));
+                    }
+
+                    let parsed_values = self.parse_formatted_value(nested)?;
+                    values.extend(parsed_values);
+                }
+                '}' => {
+                    if nested > 0 {
+                        break;
+                    }
+                    self.next_char();
+                    if let Some('}') = self.peek() {
+                        self.next_char();
+                        content.push('}');
+                    } else {
+                        return Err(SingleRbrace.to_lexical_error(self.get_pos()));
+                    }
+                }
+                '\\' if !self.kind.is_raw() => {
+                    self.next_char();
+                    content.push_str(&self.parse_escaped_char()?);
+                }
+                _ => {
+                    content.push(ch);
+                    self.next_char();
+                }
+            }
+        }
+
+        if !content.is_empty() {
+            values.push(self.expr(ExprKind::Constant {
+                value: content.into(),
+                kind: None,
+            }))
+        }
+
+        Ok(values)
+    }
+
+    pub fn parse_bytes(&mut self) -> Result<Expr, LexicalError> {
+        let mut content = String::new();
+        while let Some(ch) = self.next_char() {
+            match ch {
+                '\\' if !self.kind.is_raw() => {
+                    content.push_str(&self.parse_escaped_char()?);
+                }
+                ch => {
+                    if !ch.is_ascii() {
+                        return Err(LexicalError {
+                            error: LexicalErrorType::OtherError(
+                                "bytes can only contain ASCII literal characters".to_string(),
+                            ),
+                            location: self.get_pos(),
+                        });
+                    }
+                    content.push(ch);
+                }
+            }
+        }
+
+        Ok(self.expr(ExprKind::Constant {
+            value: Constant::Bytes(content.chars().map(|c| c as u8).collect()),
+            kind: None,
+        }))
+    }
+
+    pub fn parse_string(&mut self) -> Result<Expr, LexicalError> {
+        let mut content = String::new();
+        while let Some(ch) = self.next_char() {
+            match ch {
+                '\\' if !self.kind.is_raw() => {
+                    content.push_str(&self.parse_escaped_char()?);
+                }
+                ch => content.push(ch),
+            }
+        }
+        Ok(self.expr(ExprKind::Constant {
+            value: Constant::Str(content),
+            kind: self.kind.is_unicode().then(|| "u".to_string()),
+        }))
+    }
+
+    pub fn parse(&mut self) -> Result<Vec<Expr>, LexicalError> {
+        if self.kind.is_fstring() {
+            self.parse_fstring(0)
+        } else if self.kind.is_bytes() {
+            self.parse_bytes().map(|expr| vec![expr])
+        } else {
+            self.parse_string().map(|expr| vec![expr])
+        }
+    }
+}
+
+fn parse_fstring_expr(source: &str, location: Location) -> Result<Expr, ParseError> {
+    let fstring_body = format!("({source})");
+    parse_expression_located(
+        &fstring_body,
+        "<fstring>",
+        Location::new(location.row(), location.column() - 1),
+    )
+}
+
+pub fn parse_string(
+    source: &str,
+    kind: StringKind,
+    triple_quoted: bool,
+    start: Location,
+    end: Location,
+) -> Result<Vec<Expr>, LexicalError> {
+    StringParser::new(source, kind, triple_quoted, start, end).parse()
+}
diff --git a/parser/src/token.rs b/parser/src/token.rs
index 14ffece5..ce48410e 100644
--- a/parser/src/token.rs
+++ b/parser/src/token.rs
@@ -1,17 +1,29 @@
 //! Different token definitions.
 //! Loosely based on token.h from CPython source:
 use num_bigint::BigInt;
-use std::fmt::{self, Write};
+use std::fmt;
 
 /// Python source code can be tokenized in a sequence of these tokens.
 #[derive(Clone, Debug, PartialEq)]
 pub enum Tok {
-    Name { name: String },
-    Int { value: BigInt },
-    Float { value: f64 },
-    Complex { real: f64, imag: f64 },
-    String { value: String, kind: StringKind },
-    Bytes { value: Vec<u8> },
+    Name {
+        name: String,
+    },
+    Int {
+        value: BigInt,
+    },
+    Float {
+        value: f64,
+    },
+    Complex {
+        real: f64,
+        imag: f64,
+    },
+    String {
+        value: String,
+        kind: StringKind,
+        triple_quoted: bool,
+    },
     Newline,
     Indent,
     Dedent,
@@ -107,13 +119,6 @@ pub enum Tok {
     Yield,
 }
 
-#[derive(PartialEq, Eq, Debug, Clone)]
-pub enum StringKind {
-    Normal,
-    F,
-    U,
-}
-
 impl fmt::Display for Tok {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         use Tok::*;
@@ -122,26 +127,13 @@ impl fmt::Display for Tok {
             Int { value } => write!(f, "'{value}'"),
             Float { value } => write!(f, "'{value}'"),
             Complex { real, imag } => write!(f, "{real}j{imag}"),
-            String { value, kind } => {
-                match kind {
-                    StringKind::F => f.write_str("f")?,
-                    StringKind::U => f.write_str("u")?,
-                    StringKind::Normal => {}
-                }
-                write!(f, "{value:?}")
-            }
-            Bytes { value } => {
-                write!(f, "b\"")?;
-                for i in value {
-                    match i {
-                        9 => f.write_str("\\t")?,
-                        10 => f.write_str("\\n")?,
-                        13 => f.write_str("\\r")?,
-                        32..=126 => f.write_char(*i as char)?,
-                        _ => write!(f, "\\x{i:02x}")?,
-                    }
-                }
-                f.write_str("\"")
+            String {
+                value,
+                kind,
+                triple_quoted,
+            } => {
+                let quotes = "\"".repeat(if *triple_quoted { 3 } else { 1 });
+                write!(f, "{kind}{quotes}{value}{quotes}")
             }
             Newline => f.write_str("Newline"),
             Indent => f.write_str("Indent"),
@@ -236,3 +228,50 @@ impl fmt::Display for Tok {
         }
     }
 }
+
+#[derive(PartialEq, Eq, Debug, Clone)]
+pub enum StringKind {
+    String,
+    FString,
+    Bytes,
+    RawString,
+    RawFString,
+    RawBytes,
+    Unicode,
+}
+
+impl fmt::Display for StringKind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use StringKind::*;
+        match self {
+            String => f.write_str(""),
+            FString => f.write_str("f"),
+            Bytes => f.write_str("b"),
+            RawString => f.write_str("r"),
+            RawFString => f.write_str("rf"),
+            RawBytes => f.write_str("rb"),
+            Unicode => f.write_str("u"),
+        }
+    }
+}
+
+impl StringKind {
+    pub fn is_raw(&self) -> bool {
+        use StringKind::{RawBytes, RawFString, RawString};
+        matches!(self, RawString | RawFString | RawBytes)
+    }
+
+    pub fn is_fstring(&self) -> bool {
+        use StringKind::{FString, RawFString};
+        matches!(self, FString | RawFString)
+    }
+
+    pub fn is_bytes(&self) -> bool {
+        use StringKind::{Bytes, RawBytes};
+        matches!(self, Bytes | RawBytes)
+    }
+
+    pub fn is_unicode(&self) -> bool {
+        matches!(self, StringKind::Unicode)
+    }
+}

From b5171c3b4a583b35338ac22f24f46f46276e3f5d Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Fri, 30 Dec 2022 22:46:31 +0900
Subject: [PATCH 355/459] Fix tests

---
 ...n_codegen__compile__tests__nested_double_async_with.snap | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap b/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap
index 33180c7c..ebefb03d 100644
--- a/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap
+++ b/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap
@@ -67,11 +67,9 @@ expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyn
   9     >>   54 LoadNameAny          (3, self)
              55 LoadMethod           (fail)
              56 LoadConst            ("")
-
-  1          57 LoadNameAny          (2, stop_exc)
+             57 LoadNameAny          (2, stop_exc)
              58 FormatValue          (None)
-
-  9          59 LoadConst            (" was suppressed")
+             59 LoadConst            (" was suppressed")
              60 BuildString          (2)
              61 CallMethodPositional (1)
              62 Pop

From 6f8f655059c845c300ab4d4b6e814e1350011833 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sat, 31 Dec 2022 12:15:33 +0900
Subject: [PATCH 356/459] Move tests

---
 parser/src/fstring.rs                         | 479 ------------------
 ...__fstring_parse_selfdocumenting_base.snap} |  24 +-
 ...ring_parse_selfdocumenting_base_more.snap} |  58 +--
 ...fstring_parse_selfdocumenting_format.snap} |  36 +-
 ...g_parser__tests__parse_empty_fstring.snap} |   2 +-
 ..._string_parser__tests__parse_fstring.snap} |  28 +-
 ..._parser__tests__parse_fstring_equals.snap} |  20 +-
 ...er__tests__parse_fstring_nested_spec.snap} |  28 +-
 ...ser__tests__parse_fstring_not_equals.snap} |  20 +-
 ...tests__parse_fstring_not_nested_spec.snap} |  24 +-
 ...ts__parse_fstring_selfdoc_prec_space.snap} |  24 +-
 ...parse_fstring_selfdoc_trailing_space.snap} |  24 +-
 ...ser__tests__parse_fstring_yield_expr.snap} |  12 +-
 parser/src/string_parser.rs                   | 136 +++++
 14 files changed, 286 insertions(+), 629 deletions(-)
 delete mode 100644 parser/src/fstring.rs
 rename parser/src/snapshots/{rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap => rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap} (80%)
 rename parser/src/snapshots/{rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap => rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap} (81%)
 rename parser/src/snapshots/{rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap => rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap} (80%)
 rename parser/src/snapshots/{rustpython_parser__fstring__tests__parse_empty_fstring.snap => rustpython_parser__string_parser__tests__parse_empty_fstring.snap} (54%)
 rename parser/src/snapshots/{rustpython_parser__fstring__tests__parse_fstring.snap => rustpython_parser__string_parser__tests__parse_fstring.snap} (81%)
 rename parser/src/snapshots/{rustpython_parser__fstring__tests__parse_fstring_equals.snap => rustpython_parser__string_parser__tests__parse_fstring_equals.snap} (85%)
 rename parser/src/snapshots/{rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap => rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap} (82%)
 rename parser/src/snapshots/{rustpython_parser__fstring__tests__parse_fstring_not_equals.snap => rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap} (85%)
 rename parser/src/snapshots/{rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap => rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap} (80%)
 rename parser/src/snapshots/{rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap => rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap} (80%)
 rename parser/src/snapshots/{rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap => rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap} (80%)
 rename parser/src/snapshots/{rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap => rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap} (79%)

diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs
deleted file mode 100644
index 58e8f98f..00000000
--- a/parser/src/fstring.rs
+++ /dev/null
@@ -1,479 +0,0 @@
-// We no longer need this file
-use self::FStringErrorType::*;
-use crate::{
-    ast::{Constant, ConversionFlag, Expr, ExprKind, Location},
-    error::{FStringError, FStringErrorType, ParseError},
-    parser::parse_expression,
-};
-use std::{iter, mem, str};
-
-struct FStringParser<'a> {
-    chars: iter::Peekable<str::Chars<'a>>,
-    str_start: Location,
-    str_end: Location,
-}
-
-impl<'a> FStringParser<'a> {
-    fn new(source: &'a str, str_start: Location, str_end: Location) -> Self {
-        Self {
-            chars: source.chars().peekable(),
-            str_start,
-            str_end,
-        }
-    }
-
-    fn next_char(&mut self) -> Option<char> {
-        self.chars.next()
-    }
-
-    fn peek(&mut self) -> Option<&char> {
-        self.chars.peek()
-    }
-
-    #[inline]
-    fn expr(&self, node: ExprKind) -> Expr {
-        Expr::new(self.str_start, self.str_end, node)
-    }
-
-    fn parse_formatted_value(&mut self, nested: u8) -> Result<Vec<Expr>, FStringErrorType> {
-        let mut expression = String::new();
-        let mut spec = None;
-        let mut delims = Vec::new();
-        let mut conversion = ConversionFlag::None;
-        let mut self_documenting = false;
-        let mut trailing_seq = String::new();
-
-        while let Some(ch) = self.next_char() {
-            match ch {
-                // can be integrated better with the remainign code, but as a starting point ok
-                // in general I would do here a tokenizing of the fstrings to omit this peeking.
-                '!' if self.peek() == Some(&'=') => {
-                    expression.push_str("!=");
-                    self.next_char();
-                }
-
-                '=' if self.peek() == Some(&'=') => {
-                    expression.push_str("==");
-                    self.next_char();
-                }
-
-                '>' if self.peek() == Some(&'=') => {
-                    expression.push_str(">=");
-                    self.next_char();
-                }
-
-                '<' if self.peek() == Some(&'=') => {
-                    expression.push_str("<=");
-                    self.next_char();
-                }
-
-                '!' if delims.is_empty() && self.peek() != Some(&'=') => {
-                    if expression.trim().is_empty() {
-                        return Err(EmptyExpression);
-                    }
-
-                    conversion = match self.next_char() {
-                        Some('s') => ConversionFlag::Str,
-                        Some('a') => ConversionFlag::Ascii,
-                        Some('r') => ConversionFlag::Repr,
-                        Some(_) => {
-                            return Err(if expression.trim().is_empty() {
-                                EmptyExpression
-                            } else {
-                                InvalidConversionFlag
-                            });
-                        }
-                        None => {
-                            return Err(if expression.trim().is_empty() {
-                                EmptyExpression
-                            } else {
-                                UnclosedLbrace
-                            });
-                        }
-                    };
-
-                    if let Some(&peek) = self.peek() {
-                        if peek != '}' && peek != ':' {
-                            return Err(if expression.trim().is_empty() {
-                                EmptyExpression
-                            } else {
-                                UnclosedLbrace
-                            });
-                        }
-                    } else {
-                        return Err(if expression.trim().is_empty() {
-                            EmptyExpression
-                        } else {
-                            UnclosedLbrace
-                        });
-                    }
-                }
-
-                // match a python 3.8 self documenting expression
-                // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}'
-                '=' if self.peek() != Some(&'=') && delims.is_empty() => {
-                    self_documenting = true;
-                }
-
-                ':' if delims.is_empty() => {
-                    let parsed_spec = self.parse_spec(nested)?;
-
-                    spec = Some(Box::new(self.expr(ExprKind::JoinedStr {
-                        values: parsed_spec,
-                    })));
-                }
-                '(' | '{' | '[' => {
-                    expression.push(ch);
-                    delims.push(ch);
-                }
-                ')' => {
-                    let last_delim = delims.pop();
-                    match last_delim {
-                        Some('(') => {
-                            expression.push(ch);
-                        }
-                        Some(c) => {
-                            return Err(MismatchedDelimiter(c, ')'));
-                        }
-                        None => {
-                            return Err(Unmatched(')'));
-                        }
-                    }
-                }
-                ']' => {
-                    let last_delim = delims.pop();
-                    match last_delim {
-                        Some('[') => {
-                            expression.push(ch);
-                        }
-                        Some(c) => {
-                            return Err(MismatchedDelimiter(c, ']'));
-                        }
-                        None => {
-                            return Err(Unmatched(']'));
-                        }
-                    }
-                }
-                '}' if !delims.is_empty() => {
-                    let last_delim = delims.pop();
-                    match last_delim {
-                        Some('{') => {
-                            expression.push(ch);
-                        }
-                        Some(c) => return Err(MismatchedDelimiter(c, '}')),
-                        None => {}
-                    }
-                }
-                '}' => {
-                    if expression.trim().is_empty() {
-                        return Err(EmptyExpression);
-                    }
-
-                    let ret = if !self_documenting {
-                        vec![self.expr(ExprKind::FormattedValue {
-                            value: Box::new(
-                                parse_fstring_expr(&expression)
-                                    .map_err(|e| InvalidExpression(Box::new(e.error)))?,
-                            ),
-                            conversion: conversion as _,
-                            format_spec: spec,
-                        })]
-                    } else {
-                        vec![
-                            self.expr(ExprKind::Constant {
-                                value: Constant::Str(expression.to_owned() + "="),
-                                kind: None,
-                            }),
-                            self.expr(ExprKind::Constant {
-                                value: trailing_seq.into(),
-                                kind: None,
-                            }),
-                            self.expr(ExprKind::FormattedValue {
-                                value: Box::new(
-                                    parse_fstring_expr(&expression)
-                                        .map_err(|e| InvalidExpression(Box::new(e.error)))?,
-                                ),
-                                conversion: (if conversion == ConversionFlag::None && spec.is_none()
-                                {
-                                    ConversionFlag::Repr
-                                } else {
-                                    conversion
-                                }) as _,
-                                format_spec: spec,
-                            }),
-                        ]
-                    };
-                    return Ok(ret);
-                }
-                '"' | '\'' => {
-                    expression.push(ch);
-                    loop {
-                        let Some(c) = self.next_char() else {
-                            return Err(UnterminatedString);
-                        };
-                        expression.push(c);
-                        if c == ch {
-                            break;
-                        }
-                    }
-                }
-                ' ' if self_documenting => {
-                    trailing_seq.push(ch);
-                }
-                '\\' => return Err(ExpressionCannotInclude('\\')),
-                _ => {
-                    if self_documenting {
-                        return Err(UnclosedLbrace);
-                    }
-
-                    expression.push(ch);
-                }
-            }
-        }
-        Err(if expression.trim().is_empty() {
-            EmptyExpression
-        } else {
-            UnclosedLbrace
-        })
-    }
-
-    fn parse_spec(&mut self, nested: u8) -> Result<Vec<Expr>, FStringErrorType> {
-        let mut spec_constructor = Vec::new();
-        let mut constant_piece = String::new();
-        while let Some(&next) = self.peek() {
-            match next {
-                '{' => {
-                    if !constant_piece.is_empty() {
-                        spec_constructor.push(self.expr(ExprKind::Constant {
-                            value: constant_piece.to_owned().into(),
-                            kind: None,
-                        }));
-                        constant_piece.clear();
-                    }
-                    let parsed_expr = self.parse(nested + 1)?;
-                    spec_constructor.extend(parsed_expr);
-                    continue;
-                }
-                '}' => {
-                    break;
-                }
-                _ => {
-                    constant_piece.push(next);
-                }
-            }
-            self.next_char();
-        }
-        if !constant_piece.is_empty() {
-            spec_constructor.push(self.expr(ExprKind::Constant {
-                value: constant_piece.to_owned().into(),
-                kind: None,
-            }));
-            constant_piece.clear();
-        }
-        Ok(spec_constructor)
-    }
-
-    fn parse(&mut self, nested: u8) -> Result<Vec<Expr>, FStringErrorType> {
-        if nested >= 2 {
-            return Err(ExpressionNestedTooDeeply);
-        }
-
-        let mut content = String::new();
-        let mut values = vec![];
-
-        while let Some(&ch) = self.peek() {
-            match ch {
-                '{' => {
-                    self.next_char();
-                    if nested == 0 {
-                        match self.peek() {
-                            Some('{') => {
-                                self.next_char();
-                                content.push('{');
-                                continue;
-                            }
-                            None => return Err(UnclosedLbrace),
-                            _ => {}
-                        }
-                    }
-                    if !content.is_empty() {
-                        values.push(self.expr(ExprKind::Constant {
-                            value: mem::take(&mut content).into(),
-                            kind: None,
-                        }));
-                    }
-
-                    let parsed_values = self.parse_formatted_value(nested)?;
-                    values.extend(parsed_values);
-                }
-                '}' => {
-                    if nested > 0 {
-                        break;
-                    }
-                    self.next_char();
-                    if let Some('}') = self.peek() {
-                        self.next_char();
-                        content.push('}');
-                    } else {
-                        return Err(SingleRbrace);
-                    }
-                }
-                _ => {
-                    content.push(ch);
-                    self.next_char();
-                }
-            }
-        }
-
-        if !content.is_empty() {
-            values.push(self.expr(ExprKind::Constant {
-                value: content.into(),
-                kind: None,
-            }))
-        }
-
-        Ok(values)
-    }
-}
-
-fn parse_fstring_expr(source: &str) -> Result<Expr, ParseError> {
-    let fstring_body = format!("({source})");
-    parse_expression(&fstring_body, "<fstring>")
-}
-
-/// Parse an fstring from a string, located at a certain position in the sourcecode.
-/// In case of errors, we will get the location and the error returned.
-pub fn parse_located_fstring(
-    source: &str,
-    start: Location,
-    end: Location,
-) -> Result<Vec<Expr>, FStringError> {
-    FStringParser::new(source, start, end)
-        .parse(0)
-        .map_err(|error| FStringError {
-            error,
-            location: start,
-        })
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn parse_fstring(source: &str) -> Result<Vec<Expr>, FStringErrorType> {
-        FStringParser::new(source, Location::default(), Location::default()).parse(0)
-    }
-
-    #[test]
-    fn test_parse_fstring() {
-        let source = "{a}{ b }{{foo}}";
-        let parse_ast = parse_fstring(source).unwrap();
-
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_fstring_nested_spec() {
-        let source = "{foo:{spec}}";
-        let parse_ast = parse_fstring(source).unwrap();
-
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_fstring_not_nested_spec() {
-        let source = "{foo:spec}";
-        let parse_ast = parse_fstring(source).unwrap();
-
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_empty_fstring() {
-        insta::assert_debug_snapshot!(parse_fstring("").unwrap());
-    }
-
-    #[test]
-    fn test_fstring_parse_selfdocumenting_base() {
-        let src = "{user=}";
-        let parse_ast = parse_fstring(src).unwrap();
-
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_fstring_parse_selfdocumenting_base_more() {
-        let src = "mix {user=} with text and {second=}";
-        let parse_ast = parse_fstring(src).unwrap();
-
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_fstring_parse_selfdocumenting_format() {
-        let src = "{user=:>10}";
-        let parse_ast = parse_fstring(src).unwrap();
-
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_invalid_fstring() {
-        assert_eq!(parse_fstring("{5!a"), Err(UnclosedLbrace));
-        assert_eq!(parse_fstring("{5!a1}"), Err(UnclosedLbrace));
-        assert_eq!(parse_fstring("{5!"), Err(UnclosedLbrace));
-        assert_eq!(parse_fstring("abc{!a 'cat'}"), Err(EmptyExpression));
-        assert_eq!(parse_fstring("{!a"), Err(EmptyExpression));
-        assert_eq!(parse_fstring("{ !a}"), Err(EmptyExpression));
-
-        assert_eq!(parse_fstring("{5!}"), Err(InvalidConversionFlag));
-        assert_eq!(parse_fstring("{5!x}"), Err(InvalidConversionFlag));
-
-        assert_eq!(parse_fstring("{a:{a:{b}}}"), Err(ExpressionNestedTooDeeply));
-
-        assert_eq!(parse_fstring("{a:b}}"), Err(SingleRbrace));
-        assert_eq!(parse_fstring("}"), Err(SingleRbrace));
-        assert_eq!(parse_fstring("{a:{b}"), Err(UnclosedLbrace));
-        assert_eq!(parse_fstring("{"), Err(UnclosedLbrace));
-
-        assert_eq!(parse_fstring("{}"), Err(EmptyExpression));
-
-        // TODO: check for InvalidExpression enum?
-        assert!(parse_fstring("{class}").is_err());
-    }
-
-    #[test]
-    fn test_parse_fstring_not_equals() {
-        let source = "{1 != 2}";
-        let parse_ast = parse_fstring(source).unwrap();
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_fstring_equals() {
-        let source = "{42 == 42}";
-        let parse_ast = parse_fstring(source).unwrap();
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_fstring_selfdoc_prec_space() {
-        let source = "{x   =}";
-        let parse_ast = parse_fstring(source).unwrap();
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_fstring_selfdoc_trailing_space() {
-        let source = "{x=   }";
-        let parse_ast = parse_fstring(source).unwrap();
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_fstring_yield_expr() {
-        let source = "{yield}";
-        let parse_ast = parse_fstring(source).unwrap();
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-}
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap
similarity index 80%
rename from parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
rename to parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap
index 0941873f..de77be84 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap
+++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap
@@ -1,17 +1,17 @@
 ---
-source: compiler/parser/src/fstring.rs
+source: compiler/parser/src/string_parser.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 10,
             },
         ),
         custom: (),
@@ -24,13 +24,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 10,
             },
         ),
         custom: (),
@@ -43,13 +43,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 10,
             },
         ),
         custom: (),
@@ -57,12 +57,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 5,
+                        column: 7,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap
similarity index 81%
rename from parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
rename to parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap
index 92a05f21..e7f5bff8 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap
+++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap
@@ -1,17 +1,17 @@
 ---
-source: compiler/parser/src/fstring.rs
+source: compiler/parser/src/string_parser.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 38,
             },
         ),
         custom: (),
@@ -24,13 +24,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 38,
             },
         ),
         custom: (),
@@ -43,13 +43,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 38,
             },
         ),
         custom: (),
@@ -62,13 +62,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 38,
             },
         ),
         custom: (),
@@ -76,12 +76,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 7,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 5,
+                        column: 11,
                     },
                 ),
                 custom: (),
@@ -96,13 +96,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 38,
             },
         ),
         custom: (),
@@ -115,13 +115,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 38,
             },
         ),
         custom: (),
@@ -134,13 +134,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 38,
             },
         ),
         custom: (),
@@ -153,13 +153,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 38,
             },
         ),
         custom: (),
@@ -167,12 +167,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 29,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 7,
+                        column: 35,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap
similarity index 80%
rename from parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
rename to parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap
index ab5ea8cd..5efa6f6c 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap
+++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap
@@ -1,17 +1,17 @@
 ---
-source: compiler/parser/src/fstring.rs
+source: compiler/parser/src/string_parser.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 14,
             },
         ),
         custom: (),
@@ -24,13 +24,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 14,
             },
         ),
         custom: (),
@@ -43,13 +43,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 14,
             },
         ),
         custom: (),
@@ -57,12 +57,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 5,
+                        column: 7,
                     },
                 ),
                 custom: (),
@@ -75,13 +75,13 @@ expression: parse_ast
             format_spec: Some(
                 Located {
                     location: Location {
-                        row: 0,
+                        row: 1,
                         column: 0,
                     },
                     end_location: Some(
                         Location {
-                            row: 0,
-                            column: 0,
+                            row: 1,
+                            column: 14,
                         },
                     ),
                     custom: (),
@@ -89,13 +89,13 @@ expression: parse_ast
                         values: [
                             Located {
                                 location: Location {
-                                    row: 0,
+                                    row: 1,
                                     column: 0,
                                 },
                                 end_location: Some(
                                     Location {
-                                        row: 0,
-                                        column: 0,
+                                        row: 1,
+                                        column: 14,
                                     },
                                 ),
                                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_empty_fstring.snap
similarity index 54%
rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap
rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_empty_fstring.snap
index e561a7f9..fbc08b4a 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap
+++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_empty_fstring.snap
@@ -1,5 +1,5 @@
 ---
-source: compiler/parser/src/fstring.rs
+source: compiler/parser/src/string_parser.rs
 expression: "parse_fstring(\"\").unwrap()"
 ---
 []
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring.snap
similarity index 81%
rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring.snap
index 92428aae..32b63f95 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap
+++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring.snap
@@ -1,17 +1,17 @@
 ---
-source: compiler/parser/src/fstring.rs
+source: compiler/parser/src/string_parser.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 18,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 2,
+                        column: 4,
                     },
                 ),
                 custom: (),
@@ -39,13 +39,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 18,
             },
         ),
         custom: (),
@@ -53,12 +53,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 2,
+                    column: 7,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 3,
+                        column: 8,
                     },
                 ),
                 custom: (),
@@ -73,13 +73,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 18,
             },
         ),
         custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_equals.snap
similarity index 85%
rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_equals.snap
index fbbe8492..e9d8120a 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_equals.snap
@@ -1,17 +1,17 @@
 ---
-source: compiler/parser/src/fstring.rs
+source: compiler/parser/src/string_parser.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 13,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 9,
+                        column: 11,
                     },
                 ),
                 custom: (),
@@ -32,12 +32,12 @@ expression: parse_ast
                     left: Located {
                         location: Location {
                             row: 1,
-                            column: 1,
+                            column: 3,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 3,
+                                column: 5,
                             },
                         ),
                         custom: (),
@@ -55,12 +55,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 7,
+                                column: 9,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 9,
+                                    column: 11,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap
similarity index 82%
rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap
index ce413379..268a16e9 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap
@@ -1,17 +1,17 @@
 ---
-source: compiler/parser/src/fstring.rs
+source: compiler/parser/src/string_parser.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 15,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 4,
+                        column: 6,
                     },
                 ),
                 custom: (),
@@ -37,13 +37,13 @@ expression: parse_ast
             format_spec: Some(
                 Located {
                     location: Location {
-                        row: 0,
+                        row: 1,
                         column: 0,
                     },
                     end_location: Some(
                         Location {
-                            row: 0,
-                            column: 0,
+                            row: 1,
+                            column: 15,
                         },
                     ),
                     custom: (),
@@ -51,13 +51,13 @@ expression: parse_ast
                         values: [
                             Located {
                                 location: Location {
-                                    row: 0,
+                                    row: 1,
                                     column: 0,
                                 },
                                 end_location: Some(
                                     Location {
-                                        row: 0,
-                                        column: 0,
+                                        row: 1,
+                                        column: 15,
                                     },
                                 ),
                                 custom: (),
@@ -65,12 +65,12 @@ expression: parse_ast
                                     value: Located {
                                         location: Location {
                                             row: 1,
-                                            column: 1,
+                                            column: 8,
                                         },
                                         end_location: Some(
                                             Location {
                                                 row: 1,
-                                                column: 5,
+                                                column: 12,
                                             },
                                         ),
                                         custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap
similarity index 85%
rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap
index 8f667497..aaa2ad15 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap
@@ -1,17 +1,17 @@
 ---
-source: compiler/parser/src/fstring.rs
+source: compiler/parser/src/string_parser.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 11,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 7,
+                        column: 9,
                     },
                 ),
                 custom: (),
@@ -32,12 +32,12 @@ expression: parse_ast
                     left: Located {
                         location: Location {
                             row: 1,
-                            column: 1,
+                            column: 3,
                         },
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 2,
+                                column: 4,
                             },
                         ),
                         custom: (),
@@ -55,12 +55,12 @@ expression: parse_ast
                         Located {
                             location: Location {
                                 row: 1,
-                                column: 6,
+                                column: 8,
                             },
                             end_location: Some(
                                 Location {
                                     row: 1,
-                                    column: 7,
+                                    column: 9,
                                 },
                             ),
                             custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap
similarity index 80%
rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap
index b76f7aa4..0d1107ec 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap
@@ -1,17 +1,17 @@
 ---
-source: compiler/parser/src/fstring.rs
+source: compiler/parser/src/string_parser.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 13,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 4,
+                        column: 6,
                     },
                 ),
                 custom: (),
@@ -37,13 +37,13 @@ expression: parse_ast
             format_spec: Some(
                 Located {
                     location: Location {
-                        row: 0,
+                        row: 1,
                         column: 0,
                     },
                     end_location: Some(
                         Location {
-                            row: 0,
-                            column: 0,
+                            row: 1,
+                            column: 13,
                         },
                     ),
                     custom: (),
@@ -51,13 +51,13 @@ expression: parse_ast
                         values: [
                             Located {
                                 location: Location {
-                                    row: 0,
+                                    row: 1,
                                     column: 0,
                                 },
                                 end_location: Some(
                                     Location {
-                                        row: 0,
-                                        column: 0,
+                                        row: 1,
+                                        column: 13,
                                     },
                                 ),
                                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap
similarity index 80%
rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap
index c40f7e37..7851495f 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap
+++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap
@@ -1,17 +1,17 @@
 ---
-source: compiler/parser/src/fstring.rs
+source: compiler/parser/src/string_parser.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 10,
             },
         ),
         custom: (),
@@ -24,13 +24,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 10,
             },
         ),
         custom: (),
@@ -43,13 +43,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 10,
             },
         ),
         custom: (),
@@ -57,12 +57,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 2,
+                        column: 4,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap
similarity index 80%
rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap
index d73746a8..621e4990 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap
+++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap
@@ -1,17 +1,17 @@
 ---
-source: compiler/parser/src/fstring.rs
+source: compiler/parser/src/string_parser.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 10,
             },
         ),
         custom: (),
@@ -24,13 +24,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 10,
             },
         ),
         custom: (),
@@ -43,13 +43,13 @@ expression: parse_ast
     },
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 10,
             },
         ),
         custom: (),
@@ -57,12 +57,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 2,
+                        column: 4,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap
similarity index 79%
rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap
index 767ed51b..62a0cf43 100644
--- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap
+++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap
@@ -1,17 +1,17 @@
 ---
-source: compiler/parser/src/fstring.rs
+source: compiler/parser/src/string_parser.rs
 expression: parse_ast
 ---
 [
     Located {
         location: Location {
-            row: 0,
+            row: 1,
             column: 0,
         },
         end_location: Some(
             Location {
-                row: 0,
-                column: 0,
+                row: 1,
+                column: 10,
             },
         ),
         custom: (),
@@ -19,12 +19,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 1,
-                    column: 1,
+                    column: 3,
                 },
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 6,
+                        column: 8,
                     },
                 ),
                 custom: (),
diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
index 610951ac..1889b3f9 100644
--- a/parser/src/string_parser.rs
+++ b/parser/src/string_parser.rs
@@ -36,6 +36,7 @@ impl<'a> StringParser<'a> {
         }
     }
 
+    #[inline]
     fn next_char(&mut self) -> Option<char> {
         let Some(c) = self.chars.next() else {
             return None
@@ -48,10 +49,12 @@ impl<'a> StringParser<'a> {
         Some(c)
     }
 
+    #[inline]
     fn peek(&mut self) -> Option<&char> {
         self.chars.peek()
     }
 
+    #[inline]
     fn get_pos(&self) -> Location {
         self.location
     }
@@ -560,3 +563,136 @@ pub fn parse_string(
 ) -> Result<Vec<Expr>, LexicalError> {
     StringParser::new(source, kind, triple_quoted, start, end).parse()
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn parse_fstring(source: &str) -> Result<Vec<Expr>, FStringErrorType> {
+        StringParser::new(
+            source,
+            StringKind::FString,
+            false,
+            Location::new(1, 0),
+            Location::new(1, source.len() + 3), // 3 for prefix and quotes
+        )
+        .parse()
+        .map_err(|e| match e.error {
+            LexicalErrorType::FStringError(e) => e,
+            e => unreachable!("Unexpected error type {:?}", e),
+        })
+    }
+
+    #[test]
+    fn test_parse_fstring() {
+        let source = "{a}{ b }{{foo}}";
+        let parse_ast = parse_fstring(source).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_nested_spec() {
+        let source = "{foo:{spec}}";
+        let parse_ast = parse_fstring(source).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_not_nested_spec() {
+        let source = "{foo:spec}";
+        let parse_ast = parse_fstring(source).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_empty_fstring() {
+        insta::assert_debug_snapshot!(parse_fstring("").unwrap());
+    }
+
+    #[test]
+    fn test_fstring_parse_selfdocumenting_base() {
+        let src = "{user=}";
+        let parse_ast = parse_fstring(src).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_fstring_parse_selfdocumenting_base_more() {
+        let src = "mix {user=} with text and {second=}";
+        let parse_ast = parse_fstring(src).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_fstring_parse_selfdocumenting_format() {
+        let src = "{user=:>10}";
+        let parse_ast = parse_fstring(src).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_invalid_fstring() {
+        assert_eq!(parse_fstring("{5!a"), Err(UnclosedLbrace));
+        assert_eq!(parse_fstring("{5!a1}"), Err(UnclosedLbrace));
+        assert_eq!(parse_fstring("{5!"), Err(UnclosedLbrace));
+        assert_eq!(parse_fstring("abc{!a 'cat'}"), Err(EmptyExpression));
+        assert_eq!(parse_fstring("{!a"), Err(EmptyExpression));
+        assert_eq!(parse_fstring("{ !a}"), Err(EmptyExpression));
+
+        assert_eq!(parse_fstring("{5!}"), Err(InvalidConversionFlag));
+        assert_eq!(parse_fstring("{5!x}"), Err(InvalidConversionFlag));
+
+        assert_eq!(parse_fstring("{a:{a:{b}}}"), Err(ExpressionNestedTooDeeply));
+
+        assert_eq!(parse_fstring("{a:b}}"), Err(SingleRbrace));
+        assert_eq!(parse_fstring("}"), Err(SingleRbrace));
+        assert_eq!(parse_fstring("{a:{b}"), Err(UnclosedLbrace));
+        assert_eq!(parse_fstring("{"), Err(UnclosedLbrace));
+
+        assert_eq!(parse_fstring("{}"), Err(EmptyExpression));
+
+        // TODO: check for InvalidExpression enum?
+        assert!(parse_fstring("{class}").is_err());
+    }
+
+    #[test]
+    fn test_parse_fstring_not_equals() {
+        let source = "{1 != 2}";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_equals() {
+        let source = "{42 == 42}";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_selfdoc_prec_space() {
+        let source = "{x   =}";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_selfdoc_trailing_space() {
+        let source = "{x=   }";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_yield_expr() {
+        let source = "{yield}";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+}

From 95f3fa5a21fa3241acdd3984582c6a35d3973b59 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sat, 31 Dec 2022 23:32:08 +0900
Subject: [PATCH 357/459] Fix NamedExpr location

---
 parser/python.lalrpop                                |  4 ++--
 ...on_parser__context__tests__assign_named_expr.snap |  2 +-
 ...rse_named_expression_generator_comprehension.snap |  2 +-
 ...stpython_parser__with__tests__with_statement.snap | 12 ++++++------
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 0bee50de..9e6dd988 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -767,10 +767,10 @@ NamedExpressionTest: ast::Expr = {
 }
 
 NamedExpression: ast::Expr = {
-    <location:@L> <id:Identifier> ":=" <value:Test<"all">> <end_location:@R> => {
+    <location:@L> <id:Identifier> <end_location:@R> ":=" <value:Test<"all">> => {
         ast::Expr {
             location,
-            end_location: Some(end_location),
+            end_location: value.end_location,
             custom: (),
             node: ast::ExprKind::NamedExpr {
                 target: Box::new(ast::Expr::new(
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap
index 68d212a4..64ddb632 100644
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap
+++ b/parser/src/snapshots/rustpython_parser__context__tests__assign_named_expr.snap
@@ -37,7 +37,7 @@ expression: parse_ast
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 8,
+                                column: 4,
                             },
                         ),
                         custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
index 3bbe6c1a..639bf21f 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap
@@ -36,7 +36,7 @@ Located {
                     end_location: Some(
                         Location {
                             row: 1,
-                            column: 11,
+                            column: 2,
                         },
                     ),
                     custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
index 33cd37c4..bee91384 100644
--- a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
+++ b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
@@ -1616,7 +1616,7 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                 end_location: Some(
                                     Location {
                                         row: 19,
-                                        column: 12,
+                                        column: 7,
                                     },
                                 ),
                                 custom: (),
@@ -1704,7 +1704,7 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                 end_location: Some(
                                     Location {
                                         row: 20,
-                                        column: 12,
+                                        column: 7,
                                     },
                                 ),
                                 custom: (),
@@ -1824,7 +1824,7 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                             end_location: Some(
                                                 Location {
                                                     row: 21,
-                                                    column: 12,
+                                                    column: 7,
                                                 },
                                             ),
                                             custom: (),
@@ -1875,7 +1875,7 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                             end_location: Some(
                                                 Location {
                                                     row: 21,
-                                                    column: 20,
+                                                    column: 15,
                                                 },
                                             ),
                                             custom: (),
@@ -1981,7 +1981,7 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                             end_location: Some(
                                                 Location {
                                                     row: 22,
-                                                    column: 12,
+                                                    column: 7,
                                                 },
                                             ),
                                             custom: (),
@@ -2032,7 +2032,7 @@ expression: "parse_program(source, \"<test>\").unwrap()"
                                             end_location: Some(
                                                 Location {
                                                     row: 22,
-                                                    column: 20,
+                                                    column: 15,
                                                 },
                                             ),
                                             custom: (),

From 696fe66056106e8e62c359b50146a11adb58cc11 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 1 Jan 2023 17:43:25 +0900
Subject: [PATCH 358/459] Remove unreachable code

---
 parser/src/string.rs | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/parser/src/string.rs b/parser/src/string.rs
index 817d0e88..027d4a99 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -111,18 +111,11 @@ pub fn parse_strings(
         deduped.push(take_current(&mut current));
     }
 
-    Ok(if has_fstring {
-        Expr::new(
-            initial_start,
-            last_end,
-            ExprKind::JoinedStr { values: deduped },
-        )
-    } else {
-        deduped
-            .into_iter()
-            .exactly_one()
-            .expect("String must be concatenated to a single element.")
-    })
+    Ok(Expr::new(
+        initial_start,
+        last_end,
+        ExprKind::JoinedStr { values: deduped },
+    ))
 }
 
 #[cfg(test)]

From bf715ab1cac983ef1ffea638b56ffb10fd8f3c71 Mon Sep 17 00:00:00 2001
From: Dimitris Fasarakis Hilliard <d.f.hilliard@gmail.com>
Date: Sun, 1 Jan 2023 21:36:07 +0200
Subject: [PATCH 359/459] Move tests for with into parser.

---
 parser/src/lib.rs                             |  1 -
 parser/src/parser.rs                          | 57 +++++++++++++++++
 ...arser__parser__tests__with_statement.snap} |  2 +-
 parser/src/with.rs                            | 61 -------------------
 4 files changed, 58 insertions(+), 63 deletions(-)
 rename parser/src/snapshots/{rustpython_parser__with__tests__with_statement.snap => rustpython_parser__parser__tests__with_statement.snap} (99%)
 delete mode 100644 parser/src/with.rs

diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index d266d2a6..ce9dde1d 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -33,4 +33,3 @@ mod python;
 mod context;
 mod string;
 pub mod token;
-mod with;
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 6dcbaccd..aad02882 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -259,4 +259,61 @@ class Foo(A, B):
         let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
+
+    #[test]
+    fn test_with_statement() {
+        let source = "\
+with 0: pass
+with 0 as x: pass
+with 0, 1: pass
+with 0 as x, 1 as y: pass
+with 0 if 1 else 2: pass
+with 0 if 1 else 2 as x: pass
+with (): pass
+with () as x: pass
+with (0): pass
+with (0) as x: pass
+with (0,): pass
+with (0,) as x: pass
+with (0, 1): pass
+with (0, 1) as x: pass
+with (*a,): pass
+with (*a,) as x: pass
+with (0, *a): pass
+with (0, *a) as x: pass
+with (a := 0): pass
+with (a := 0) as x: pass
+with (a := 0, b := 1): pass
+with (a := 0, b := 1) as x: pass
+with (0 as a): pass
+with (0 as a,): pass
+with (0 as a, 1 as b): pass
+with (0 as a, 1 as b,): pass
+";
+        insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
+    }
+
+    #[test]
+    fn test_with_statement_invalid() {
+        for source in [
+            "with 0,: pass",
+            "with 0 as x,: pass",
+            "with 0 as *x: pass",
+            "with *a: pass",
+            "with *a as x: pass",
+            "with (*a): pass",
+            "with (*a) as x: pass",
+            "with *a, 0 as x: pass",
+            "with (*a, 0 as x): pass",
+            "with 0 as x, *a: pass",
+            "with (0 as x, *a): pass",
+            "with (0 as x) as y: pass",
+            "with (0 as x), 1: pass",
+            "with ((0 as x)): pass",
+            "with a := 0 as x: pass",
+            "with (a := 0 as x): pass",
+        ] {
+            assert!(parse_program(source, "<test>").is_err());
+        }
+    }
 }
diff --git a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap b/parser/src/snapshots/rustpython_parser__parser__tests__with_statement.snap
similarity index 99%
rename from parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
rename to parser/src/snapshots/rustpython_parser__parser__tests__with_statement.snap
index bee91384..8ac056bb 100644
--- a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__with_statement.snap
@@ -1,5 +1,5 @@
 ---
-source: compiler/parser/src/with.rs
+source: compiler/parser/src/parser.rs
 expression: "parse_program(source, \"<test>\").unwrap()"
 ---
 [
diff --git a/parser/src/with.rs b/parser/src/with.rs
deleted file mode 100644
index b05fb373..00000000
--- a/parser/src/with.rs
+++ /dev/null
@@ -1,61 +0,0 @@
-#[cfg(test)]
-mod tests {
-    use crate::parser::parse_program;
-
-    #[test]
-    fn test_with_statement() {
-        let source = "\
-with 0: pass
-with 0 as x: pass
-with 0, 1: pass
-with 0 as x, 1 as y: pass
-with 0 if 1 else 2: pass
-with 0 if 1 else 2 as x: pass
-with (): pass
-with () as x: pass
-with (0): pass
-with (0) as x: pass
-with (0,): pass
-with (0,) as x: pass
-with (0, 1): pass
-with (0, 1) as x: pass
-with (*a,): pass
-with (*a,) as x: pass
-with (0, *a): pass
-with (0, *a) as x: pass
-with (a := 0): pass
-with (a := 0) as x: pass
-with (a := 0, b := 1): pass
-with (a := 0, b := 1) as x: pass
-with (0 as a): pass
-with (0 as a,): pass
-with (0 as a, 1 as b): pass
-with (0 as a, 1 as b,): pass
-";
-        insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
-    }
-
-    #[test]
-    fn test_with_statement_invalid() {
-        for source in [
-            "with 0,: pass",
-            "with 0 as x,: pass",
-            "with 0 as *x: pass",
-            "with *a: pass",
-            "with *a as x: pass",
-            "with (*a): pass",
-            "with (*a) as x: pass",
-            "with *a, 0 as x: pass",
-            "with (*a, 0 as x): pass",
-            "with 0 as x, *a: pass",
-            "with (0 as x, *a): pass",
-            "with (0 as x) as y: pass",
-            "with (0 as x), 1: pass",
-            "with ((0 as x)): pass",
-            "with a := 0 as x: pass",
-            "with (a := 0 as x): pass",
-        ] {
-            assert!(parse_program(source, "<test>").is_err());
-        }
-    }
-}

From 0c35874aecfb919ace875855f9066ed82feefa5c Mon Sep 17 00:00:00 2001
From: Bijan Naimi <bijanrnaimi@gmail.com>
Date: Sun, 1 Jan 2023 15:41:51 -0800
Subject: [PATCH 360/459] changed the shell logic for handling indents

---
 parser/src/error.rs | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/parser/src/error.rs b/parser/src/error.rs
index e1e6b2b8..091bc771 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -205,10 +205,22 @@ pub(crate) fn parse_error_from_lalrpop(
                 source_path,
             }
         }
-        LalrpopError::UnrecognizedEOF { location, .. } => ParseError {
-            error: ParseErrorType::Eof,
-            location,
-            source_path,
+        LalrpopError::UnrecognizedEOF { location, expected } => {
+            // This could be an initial indentation error that we should ignore
+            let indent_error = expected == ["Indent"];
+            if indent_error {
+                ParseError {
+                    error: ParseErrorType::Lexical(LexicalErrorType::IndentationError),
+                    location,
+                    source_path,
+                }
+            } else {
+                ParseError {
+                    error: ParseErrorType::Eof,
+                    location,
+                    source_path,
+                }
+            }
         },
     }
 }

From 5f90db1527a11966de8802ac36dc18152ec2fb4d Mon Sep 17 00:00:00 2001
From: Bijan Naimi <bijanrnaimi@gmail.com>
Date: Sun, 1 Jan 2023 17:28:49 -0800
Subject: [PATCH 361/459] forgot to add formatted errors.rs

---
 parser/src/error.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/src/error.rs b/parser/src/error.rs
index 091bc771..b6611c29 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -221,7 +221,7 @@ pub(crate) fn parse_error_from_lalrpop(
                     source_path,
                 }
             }
-        },
+        }
     }
 }
 

From ef369a926ac7d4549342d2f6e147d8fc3334e439 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Sun, 1 Jan 2023 09:09:58 +0900
Subject: [PATCH 362/459] Move (c)format basic implementations to
 rustpython-common

---
 parser/src/lexer.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index db975618..3e879d6a 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -81,7 +81,7 @@ impl Indentations {
     fn current(&self) -> &IndentationLevel {
         self.indent_stack
             .last()
-            .expect("Indetations must have at least one level")
+            .expect("Indentations must have at least one level")
     }
 }
 

From 33fd321fad3dae06d8f1bf1bf7acae2dc9a6445e Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Mon, 2 Jan 2023 20:48:40 +0900
Subject: [PATCH 363/459] Remove unreachable code in
 compiler/parser/src/string_parser.rs

---
 parser/src/string_parser.rs | 24 ++++--------------------
 1 file changed, 4 insertions(+), 20 deletions(-)

diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
index 1889b3f9..30409ccc 100644
--- a/parser/src/string_parser.rs
+++ b/parser/src/string_parser.rs
@@ -219,35 +219,19 @@ impl<'a> StringParser<'a> {
                         Some('a') => ConversionFlag::Ascii,
                         Some('r') => ConversionFlag::Repr,
                         Some(_) => {
-                            return Err(if expression.trim().is_empty() {
-                                EmptyExpression.to_lexical_error(self.get_pos())
-                            } else {
-                                InvalidConversionFlag.to_lexical_error(self.get_pos())
-                            });
+                            return Err(InvalidConversionFlag.to_lexical_error(self.get_pos()));
                         }
                         None => {
-                            return Err(if expression.trim().is_empty() {
-                                EmptyExpression.to_lexical_error(self.get_pos())
-                            } else {
-                                UnclosedLbrace.to_lexical_error(self.get_pos())
-                            });
+                            return Err(UnclosedLbrace.to_lexical_error(self.get_pos()));
                         }
                     };
 
                     if let Some(&peek) = self.peek() {
                         if peek != '}' && peek != ':' {
-                            return Err(if expression.trim().is_empty() {
-                                EmptyExpression.to_lexical_error(self.get_pos())
-                            } else {
-                                UnclosedLbrace.to_lexical_error(self.get_pos())
-                            });
+                            return Err(UnclosedLbrace.to_lexical_error(self.get_pos()));
                         }
                     } else {
-                        return Err(if expression.trim().is_empty() {
-                            EmptyExpression.to_lexical_error(self.get_pos())
-                        } else {
-                            UnclosedLbrace.to_lexical_error(self.get_pos())
-                        });
+                        return Err(UnclosedLbrace.to_lexical_error(self.get_pos()));
                     }
                 }
 

From 383ad672e74c6d62acf455cb0b2f2560fc40b768 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Mon, 2 Jan 2023 22:26:09 +0900
Subject: [PATCH 364/459] Simplify code using match

---
 parser/src/string_parser.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
index 30409ccc..4a55eb29 100644
--- a/parser/src/string_parser.rs
+++ b/parser/src/string_parser.rs
@@ -226,12 +226,12 @@ impl<'a> StringParser<'a> {
                         }
                     };
 
-                    if let Some(&peek) = self.peek() {
-                        if peek != '}' && peek != ':' {
-                            return Err(UnclosedLbrace.to_lexical_error(self.get_pos()));
+                    match self.peek() {
+                        Some('}' | ':') => {}
+                        Some(_) => {
+                            return Err(InvalidConversionFlag.to_lexical_error(self.get_pos()))
                         }
-                    } else {
-                        return Err(UnclosedLbrace.to_lexical_error(self.get_pos()));
+                        None => return Err(UnclosedLbrace.to_lexical_error(self.get_pos())),
                     }
                 }
 

From 9f6d70382b7840250b8ed81afe5dd4a938190a8e Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Mon, 2 Jan 2023 22:54:48 +0900
Subject: [PATCH 365/459] Fix match

---
 parser/src/string_parser.rs | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
index 4a55eb29..b137ef2f 100644
--- a/parser/src/string_parser.rs
+++ b/parser/src/string_parser.rs
@@ -228,10 +228,9 @@ impl<'a> StringParser<'a> {
 
                     match self.peek() {
                         Some('}' | ':') => {}
-                        Some(_) => {
-                            return Err(InvalidConversionFlag.to_lexical_error(self.get_pos()))
+                        Some(_) | None => {
+                            return Err(UnclosedLbrace.to_lexical_error(self.get_pos()))
                         }
-                        None => return Err(UnclosedLbrace.to_lexical_error(self.get_pos())),
                     }
                 }
 

From 83ebf39ecc0a369a3394f6d07bdd69e9bac0dfa2 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Mon, 2 Jan 2023 23:16:51 +0900
Subject: [PATCH 366/459] Merge match arms in parse_formatted_value

---
 parser/src/string_parser.rs | 21 +++------------------
 1 file changed, 3 insertions(+), 18 deletions(-)

diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
index 1889b3f9..8761a354 100644
--- a/parser/src/string_parser.rs
+++ b/parser/src/string_parser.rs
@@ -189,26 +189,11 @@ impl<'a> StringParser<'a> {
             match ch {
                 // can be integrated better with the remaining code, but as a starting point ok
                 // in general I would do here a tokenizing of the fstrings to omit this peeking.
-                '!' if self.peek() == Some(&'=') => {
-                    expression.push_str("!=");
-                    self.next_char();
-                }
-
-                '=' if self.peek() == Some(&'=') => {
-                    expression.push_str("==");
-                    self.next_char();
-                }
-
-                '>' if self.peek() == Some(&'=') => {
-                    expression.push_str(">=");
-                    self.next_char();
-                }
-
-                '<' if self.peek() == Some(&'=') => {
-                    expression.push_str("<=");
+                '!' | '=' | '>' | '<' if self.peek() == Some(&'=') => {
+                    expression.push(ch);
+                    expression.push('=');
                     self.next_char();
                 }
-
                 '!' if delims.is_empty() && self.peek() != Some(&'=') => {
                     if expression.trim().is_empty() {
                         return Err(EmptyExpression.to_lexical_error(self.get_pos()));

From 4c4bc4116344c79c5abde75feb292734bcda9b2b Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Tue, 3 Jan 2023 01:23:22 +0900
Subject: [PATCH 367/459] Use drain

---
 parser/src/string_parser.rs | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
index fe87245d..c131683c 100644
--- a/parser/src/string_parser.rs
+++ b/parser/src/string_parser.rs
@@ -5,7 +5,7 @@ use crate::{
     parser::parse_expression_located,
     token::StringKind,
 };
-use std::{iter, mem, str};
+use std::{iter, str};
 
 /// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
 pub const MAX_UNICODE_NAME: usize = 88;
@@ -365,10 +365,9 @@ impl<'a> StringParser<'a> {
                 '{' => {
                     if !constant_piece.is_empty() {
                         spec_constructor.push(self.expr(ExprKind::Constant {
-                            value: constant_piece.to_owned().into(),
+                            value: constant_piece.drain(..).collect::<String>().into(),
                             kind: None,
                         }));
-                        constant_piece.clear();
                     }
                     let parsed_expr = self.parse_fstring(nested + 1)?;
                     spec_constructor.extend(parsed_expr);
@@ -385,10 +384,9 @@ impl<'a> StringParser<'a> {
         }
         if !constant_piece.is_empty() {
             spec_constructor.push(self.expr(ExprKind::Constant {
-                value: constant_piece.to_owned().into(),
+                value: constant_piece.drain(..).collect::<String>().into(),
                 kind: None,
             }));
-            constant_piece.clear();
         }
         Ok(spec_constructor)
     }
@@ -418,7 +416,7 @@ impl<'a> StringParser<'a> {
                     }
                     if !content.is_empty() {
                         values.push(self.expr(ExprKind::Constant {
-                            value: mem::take(&mut content).into(),
+                            value: content.drain(..).collect::<String>().into(),
                             kind: None,
                         }));
                     }

From 6374c0df132a13880a321ca703847edd24d10357 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Tue, 3 Jan 2023 12:46:10 +0900
Subject: [PATCH 368/459] Improve error conversion in string_parsers.rs

---
 parser/src/error.rs         |  30 ++++--
 parser/src/string_parser.rs | 183 +++++++++++++++++++-----------------
 2 files changed, 120 insertions(+), 93 deletions(-)

diff --git a/parser/src/error.rs b/parser/src/error.rs
index e1e6b2b8..d2a11d70 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -12,6 +12,12 @@ pub struct LexicalError {
     pub location: Location,
 }
 
+impl LexicalError {
+    pub fn new(error: LexicalErrorType, location: Location) -> Self {
+        Self { error, location }
+    }
+}
+
 #[derive(Debug, PartialEq)]
 pub enum LexicalErrorType {
     StringError,
@@ -85,6 +91,21 @@ pub struct FStringError {
     pub location: Location,
 }
 
+impl FStringError {
+    pub fn new(error: FStringErrorType, location: Location) -> Self {
+        Self { error, location }
+    }
+}
+
+impl From<FStringError> for LexicalError {
+    fn from(err: FStringError) -> Self {
+        LexicalError {
+            error: LexicalErrorType::FStringError(err.error),
+            location: err.location,
+        }
+    }
+}
+
 #[derive(Debug, PartialEq)]
 pub enum FStringErrorType {
     UnclosedLbrace,
@@ -101,15 +122,6 @@ pub enum FStringErrorType {
     UnterminatedString,
 }
 
-impl FStringErrorType {
-    pub fn to_lexical_error(self, location: Location) -> LexicalError {
-        LexicalError {
-            error: LexicalErrorType::FStringError(self),
-            location,
-        }
-    }
-}
-
 impl fmt::Display for FStringErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
index fe87245d..5134c15a 100644
--- a/parser/src/string_parser.rs
+++ b/parser/src/string_parser.rs
@@ -1,7 +1,7 @@
 use self::FStringErrorType::*;
 use crate::{
     ast::{Constant, ConversionFlag, Expr, ExprKind, Location},
-    error::{FStringErrorType, LexicalError, LexicalErrorType, ParseError},
+    error::{FStringError, FStringErrorType, LexicalError, LexicalErrorType, ParseError},
     parser::parse_expression_located,
     token::StringKind,
 };
@@ -66,10 +66,7 @@ impl<'a> StringParser<'a> {
 
     fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
         let mut p: u32 = 0u32;
-        let unicode_error = LexicalError {
-            error: LexicalErrorType::UnicodeError,
-            location: self.get_pos(),
-        };
+        let unicode_error = LexicalError::new(LexicalErrorType::UnicodeError, self.get_pos());
         for i in 1..=literal_number {
             match self.next_char() {
                 Some(c) => match c.to_digit(16) {
@@ -103,12 +100,7 @@ impl<'a> StringParser<'a> {
         let start_pos = self.get_pos();
         match self.next_char() {
             Some('{') => {}
-            _ => {
-                return Err(LexicalError {
-                    error: LexicalErrorType::StringError,
-                    location: start_pos,
-                })
-            }
+            _ => return Err(LexicalError::new(LexicalErrorType::StringError, start_pos)),
         }
         let start_pos = self.get_pos();
         let mut name = String::new();
@@ -117,25 +109,23 @@ impl<'a> StringParser<'a> {
                 Some('}') => break,
                 Some(c) => name.push(c),
                 None => {
-                    return Err(LexicalError {
-                        error: LexicalErrorType::StringError,
-                        location: self.get_pos(),
-                    })
+                    return Err(LexicalError::new(
+                        LexicalErrorType::StringError,
+                        self.get_pos(),
+                    ))
                 }
             }
         }
 
         if name.len() > MAX_UNICODE_NAME {
-            return Err(LexicalError {
-                error: LexicalErrorType::UnicodeError,
-                location: self.get_pos(),
-            });
+            return Err(LexicalError::new(
+                LexicalErrorType::UnicodeError,
+                self.get_pos(),
+            ));
         }
 
-        unicode_names2::character(&name).ok_or(LexicalError {
-            error: LexicalErrorType::UnicodeError,
-            location: start_pos,
-        })
+        unicode_names2::character(&name)
+            .ok_or(LexicalError::new(LexicalErrorType::UnicodeError, start_pos))
     }
 
     fn parse_escaped_char(&mut self) -> Result<String, LexicalError> {
@@ -159,20 +149,20 @@ impl<'a> StringParser<'a> {
                 'N' if !self.kind.is_bytes() => self.parse_unicode_name()?.to_string(),
                 c => {
                     if self.kind.is_bytes() && !c.is_ascii() {
-                        return Err(LexicalError {
-                            error: LexicalErrorType::OtherError(
+                        return Err(LexicalError::new(
+                            LexicalErrorType::OtherError(
                                 "bytes can only contain ASCII literal characters".to_owned(),
                             ),
-                            location: self.get_pos(),
-                        });
+                            self.get_pos(),
+                        ));
                     }
                     format!("\\{c}")
                 }
             }),
-            None => Err(LexicalError {
-                error: LexicalErrorType::StringError,
-                location: self.get_pos(),
-            }),
+            None => Err(LexicalError::new(
+                LexicalErrorType::StringError,
+                self.get_pos(),
+            )),
         }
     }
 
@@ -196,7 +186,7 @@ impl<'a> StringParser<'a> {
                 }
                 '!' if delims.is_empty() && self.peek() != Some(&'=') => {
                     if expression.trim().is_empty() {
-                        return Err(EmptyExpression.to_lexical_error(self.get_pos()));
+                        return Err(FStringError::new(EmptyExpression, self.get_pos()).into());
                     }
 
                     conversion = match self.next_char() {
@@ -204,17 +194,19 @@ impl<'a> StringParser<'a> {
                         Some('a') => ConversionFlag::Ascii,
                         Some('r') => ConversionFlag::Repr,
                         Some(_) => {
-                            return Err(InvalidConversionFlag.to_lexical_error(self.get_pos()));
+                            return Err(
+                                FStringError::new(InvalidConversionFlag, self.get_pos()).into()
+                            );
                         }
                         None => {
-                            return Err(UnclosedLbrace.to_lexical_error(self.get_pos()));
+                            return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into());
                         }
                     };
 
                     match self.peek() {
                         Some('}' | ':') => {}
                         Some(_) | None => {
-                            return Err(UnclosedLbrace.to_lexical_error(self.get_pos()))
+                            return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into());
                         }
                     }
                 }
@@ -243,12 +235,14 @@ impl<'a> StringParser<'a> {
                             expression.push(ch);
                         }
                         Some(c) => {
-                            return Err(
-                                MismatchedDelimiter(c, ')').to_lexical_error(self.get_pos())
-                            );
+                            return Err(FStringError::new(
+                                MismatchedDelimiter(c, ')'),
+                                self.get_pos(),
+                            )
+                            .into());
                         }
                         None => {
-                            return Err(Unmatched(')').to_lexical_error(self.get_pos()));
+                            return Err(FStringError::new(Unmatched(')'), self.get_pos()).into());
                         }
                     }
                 }
@@ -259,12 +253,14 @@ impl<'a> StringParser<'a> {
                             expression.push(ch);
                         }
                         Some(c) => {
-                            return Err(
-                                MismatchedDelimiter(c, ']').to_lexical_error(self.get_pos())
-                            );
+                            return Err(FStringError::new(
+                                MismatchedDelimiter(c, ']'),
+                                self.get_pos(),
+                            )
+                            .into());
                         }
                         None => {
-                            return Err(Unmatched(']').to_lexical_error(self.get_pos()));
+                            return Err(FStringError::new(Unmatched(']'), self.get_pos()).into());
                         }
                     }
                 }
@@ -275,22 +271,28 @@ impl<'a> StringParser<'a> {
                             expression.push(ch);
                         }
                         Some(c) => {
-                            return Err(MismatchedDelimiter(c, '}').to_lexical_error(self.get_pos()))
+                            return Err(FStringError::new(
+                                MismatchedDelimiter(c, '}'),
+                                self.get_pos(),
+                            )
+                            .into());
                         }
                         None => {}
                     }
                 }
                 '}' => {
                     if expression.trim().is_empty() {
-                        return Err(EmptyExpression.to_lexical_error(self.get_pos()));
+                        return Err(FStringError::new(EmptyExpression, self.get_pos()).into());
                     }
 
                     let ret = if !self_documenting {
                         vec![self.expr(ExprKind::FormattedValue {
                             value: Box::new(parse_fstring_expr(&expression, location).map_err(
                                 |e| {
-                                    InvalidExpression(Box::new(e.error))
-                                        .to_lexical_error(self.get_pos())
+                                    FStringError::new(
+                                        InvalidExpression(Box::new(e.error)),
+                                        location,
+                                    )
                                 },
                             )?),
                             conversion: conversion as _,
@@ -309,8 +311,10 @@ impl<'a> StringParser<'a> {
                             self.expr(ExprKind::FormattedValue {
                                 value: Box::new(
                                     parse_fstring_expr(&expression, location).map_err(|e| {
-                                        InvalidExpression(Box::new(e.error))
-                                            .to_lexical_error(self.get_pos())
+                                        FStringError::new(
+                                            InvalidExpression(Box::new(e.error)),
+                                            location,
+                                        )
                                     })?,
                                 ),
                                 conversion: (if conversion == ConversionFlag::None && spec.is_none()
@@ -329,7 +333,7 @@ impl<'a> StringParser<'a> {
                     expression.push(ch);
                     loop {
                         let Some(c) = self.next_char() else {
-                            return Err(UnterminatedString.to_lexical_error(self.get_pos()));
+                            return Err(FStringError::new(UnterminatedString, self.get_pos()).into());
                         };
                         expression.push(c);
                         if c == ch {
@@ -340,10 +344,10 @@ impl<'a> StringParser<'a> {
                 ' ' if self_documenting => {
                     trailing_seq.push(ch);
                 }
-                '\\' => return Err(ExpressionCannotInclude('\\').to_lexical_error(self.get_pos())),
+                '\\' => return Err(FStringError::new(UnterminatedString, self.get_pos()).into()),
                 _ => {
                     if self_documenting {
-                        return Err(UnclosedLbrace.to_lexical_error(self.get_pos()));
+                        return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into());
                     }
 
                     expression.push(ch);
@@ -351,9 +355,9 @@ impl<'a> StringParser<'a> {
             }
         }
         Err(if expression.trim().is_empty() {
-            EmptyExpression.to_lexical_error(self.get_pos())
+            FStringError::new(EmptyExpression, self.get_pos()).into()
         } else {
-            UnclosedLbrace.to_lexical_error(self.get_pos())
+            FStringError::new(UnclosedLbrace, self.get_pos()).into()
         })
     }
 
@@ -395,7 +399,7 @@ impl<'a> StringParser<'a> {
 
     fn parse_fstring(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
         if nested >= 2 {
-            return Err(ExpressionNestedTooDeeply.to_lexical_error(self.get_pos()));
+            return Err(FStringError::new(ExpressionNestedTooDeeply, self.get_pos()).into());
         }
 
         let mut content = String::new();
@@ -412,7 +416,9 @@ impl<'a> StringParser<'a> {
                                 content.push('{');
                                 continue;
                             }
-                            None => return Err(UnclosedLbrace.to_lexical_error(self.get_pos())),
+                            None => {
+                                return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into())
+                            }
                             _ => {}
                         }
                     }
@@ -435,7 +441,7 @@ impl<'a> StringParser<'a> {
                         self.next_char();
                         content.push('}');
                     } else {
-                        return Err(SingleRbrace.to_lexical_error(self.get_pos()));
+                        return Err(FStringError::new(SingleRbrace, self.get_pos()).into());
                     }
                 }
                 '\\' if !self.kind.is_raw() => {
@@ -468,12 +474,12 @@ impl<'a> StringParser<'a> {
                 }
                 ch => {
                     if !ch.is_ascii() {
-                        return Err(LexicalError {
-                            error: LexicalErrorType::OtherError(
+                        return Err(LexicalError::new(
+                            LexicalErrorType::OtherError(
                                 "bytes can only contain ASCII literal characters".to_string(),
                             ),
-                            location: self.get_pos(),
-                        });
+                            self.get_pos(),
+                        ));
                     }
                     content.push(ch);
                 }
@@ -536,7 +542,7 @@ pub fn parse_string(
 mod tests {
     use super::*;
 
-    fn parse_fstring(source: &str) -> Result<Vec<Expr>, FStringErrorType> {
+    fn parse_fstring(source: &str) -> Result<Vec<Expr>, LexicalError> {
         StringParser::new(
             source,
             StringKind::FString,
@@ -545,10 +551,6 @@ mod tests {
             Location::new(1, source.len() + 3), // 3 for prefix and quotes
         )
         .parse()
-        .map_err(|e| match e.error {
-            LexicalErrorType::FStringError(e) => e,
-            e => unreachable!("Unexpected error type {:?}", e),
-        })
     }
 
     #[test]
@@ -604,26 +606,39 @@ mod tests {
         insta::assert_debug_snapshot!(parse_ast);
     }
 
+    fn parse_fstring_error(source: &str) -> FStringErrorType {
+        parse_fstring(source)
+            .map_err(|e| match e.error {
+                LexicalErrorType::FStringError(e) => e,
+                e => unreachable!("Expected FStringError: {:?}", e),
+            })
+            .err()
+            .expect("Expected error")
+    }
+
     #[test]
     fn test_parse_invalid_fstring() {
-        assert_eq!(parse_fstring("{5!a"), Err(UnclosedLbrace));
-        assert_eq!(parse_fstring("{5!a1}"), Err(UnclosedLbrace));
-        assert_eq!(parse_fstring("{5!"), Err(UnclosedLbrace));
-        assert_eq!(parse_fstring("abc{!a 'cat'}"), Err(EmptyExpression));
-        assert_eq!(parse_fstring("{!a"), Err(EmptyExpression));
-        assert_eq!(parse_fstring("{ !a}"), Err(EmptyExpression));
-
-        assert_eq!(parse_fstring("{5!}"), Err(InvalidConversionFlag));
-        assert_eq!(parse_fstring("{5!x}"), Err(InvalidConversionFlag));
-
-        assert_eq!(parse_fstring("{a:{a:{b}}}"), Err(ExpressionNestedTooDeeply));
-
-        assert_eq!(parse_fstring("{a:b}}"), Err(SingleRbrace));
-        assert_eq!(parse_fstring("}"), Err(SingleRbrace));
-        assert_eq!(parse_fstring("{a:{b}"), Err(UnclosedLbrace));
-        assert_eq!(parse_fstring("{"), Err(UnclosedLbrace));
-
-        assert_eq!(parse_fstring("{}"), Err(EmptyExpression));
+        assert_eq!(parse_fstring_error("{5!a"), UnclosedLbrace);
+        assert_eq!(parse_fstring_error("{5!a1}"), UnclosedLbrace);
+        assert_eq!(parse_fstring_error("{5!"), UnclosedLbrace);
+        assert_eq!(parse_fstring_error("abc{!a 'cat'}"), EmptyExpression);
+        assert_eq!(parse_fstring_error("{!a"), EmptyExpression);
+        assert_eq!(parse_fstring_error("{ !a}"), EmptyExpression);
+
+        assert_eq!(parse_fstring_error("{5!}"), InvalidConversionFlag);
+        assert_eq!(parse_fstring_error("{5!x}"), InvalidConversionFlag);
+
+        assert_eq!(
+            parse_fstring_error("{a:{a:{b}}}"),
+            ExpressionNestedTooDeeply
+        );
+
+        assert_eq!(parse_fstring_error("{a:b}}"), SingleRbrace);
+        assert_eq!(parse_fstring_error("}"), SingleRbrace);
+        assert_eq!(parse_fstring_error("{a:{b}"), UnclosedLbrace);
+        assert_eq!(parse_fstring_error("{"), UnclosedLbrace);
+
+        assert_eq!(parse_fstring_error("{}"), EmptyExpression);
 
         // TODO: check for InvalidExpression enum?
         assert!(parse_fstring("{class}").is_err());

From d1d22282bc299a3df145c6957f183bfac38b1ef8 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Tue, 3 Jan 2023 12:51:31 +0900
Subject: [PATCH 369/459] Fix clippy error

---
 parser/src/string_parser.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
index 5134c15a..4e76580c 100644
--- a/parser/src/string_parser.rs
+++ b/parser/src/string_parser.rs
@@ -125,7 +125,7 @@ impl<'a> StringParser<'a> {
         }
 
         unicode_names2::character(&name)
-            .ok_or(LexicalError::new(LexicalErrorType::UnicodeError, start_pos))
+            .ok_or_else(|| LexicalError::new(LexicalErrorType::UnicodeError, start_pos))
     }
 
     fn parse_escaped_char(&mut self) -> Result<String, LexicalError> {

From b6647b01715d8e6257dba0961b71c2828c02eec6 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Tue, 3 Jan 2023 13:35:47 +0900
Subject: [PATCH 370/459] Remove repetitive to_string in parse_escaped_char

---
 parser/src/string_parser.rs | 66 ++++++++++++++++++++-----------------
 1 file changed, 35 insertions(+), 31 deletions(-)

diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
index 8f33a9d9..b3054935 100644
--- a/parser/src/string_parser.rs
+++ b/parser/src/string_parser.rs
@@ -130,39 +130,43 @@ impl<'a> StringParser<'a> {
 
     fn parse_escaped_char(&mut self) -> Result<String, LexicalError> {
         match self.next_char() {
-            Some(c) => Ok(match c {
-                '\\' => '\\'.to_string(),
-                '\'' => '\''.to_string(),
-                '\"' => '"'.to_string(),
-                '\n' => "".to_string(),
-                'a' => '\x07'.to_string(),
-                'b' => '\x08'.to_string(),
-                'f' => '\x0c'.to_string(),
-                'n' => '\n'.to_string(),
-                'r' => '\r'.to_string(),
-                't' => '\t'.to_string(),
-                'v' => '\x0b'.to_string(),
-                o @ '0'..='7' => self.parse_octet(o).to_string(),
-                'x' => self.parse_unicode_literal(2)?.to_string(),
-                'u' if !self.kind.is_bytes() => self.parse_unicode_literal(4)?.to_string(),
-                'U' if !self.kind.is_bytes() => self.parse_unicode_literal(8)?.to_string(),
-                'N' if !self.kind.is_bytes() => self.parse_unicode_name()?.to_string(),
-                c => {
-                    if self.kind.is_bytes() && !c.is_ascii() {
-                        return Err(LexicalError::new(
-                            LexicalErrorType::OtherError(
-                                "bytes can only contain ASCII literal characters".to_owned(),
-                            ),
-                            self.get_pos(),
-                        ));
+            Some(c) => {
+                let char = match c {
+                    '\\' => '\\',
+                    '\'' => '\'',
+                    '\"' => '"',
+                    'a' => '\x07',
+                    'b' => '\x08',
+                    'f' => '\x0c',
+                    'n' => '\n',
+                    'r' => '\r',
+                    't' => '\t',
+                    'v' => '\x0b',
+                    o @ '0'..='7' => self.parse_octet(o),
+                    'x' => self.parse_unicode_literal(2)?,
+                    'u' if !self.kind.is_bytes() => self.parse_unicode_literal(4)?,
+                    'U' if !self.kind.is_bytes() => self.parse_unicode_literal(8)?,
+                    'N' if !self.kind.is_bytes() => self.parse_unicode_name()?,
+                    // Special cases where the escape sequence is not a single character
+                    '\n' => return Ok("".to_string()),
+                    c => {
+                        if self.kind.is_bytes() && !c.is_ascii() {
+                            return Err(LexicalError {
+                                error: LexicalErrorType::OtherError(
+                                    "bytes can only contain ASCII literal characters".to_owned(),
+                                ),
+                                location: self.get_pos(),
+                            });
+                        }
+                        return Ok(format!("\\{c}"));
                     }
-                    format!("\\{c}")
-                }
+                };
+                Ok(char.to_string())
+            }
+            None => Err(LexicalError {
+                error: LexicalErrorType::StringError,
+                location: self.get_pos(),
             }),
-            None => Err(LexicalError::new(
-                LexicalErrorType::StringError,
-                self.get_pos(),
-            )),
         }
     }
 

From 0b65609d261a9f1cf76ad28b8ea82cf4f5b40bf7 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Tue, 3 Jan 2023 17:27:35 +0900
Subject: [PATCH 371/459] Improve CharWindow

---
 parser/src/lexer.rs | 47 +++++++++++++++++++++------------------------
 1 file changed, 22 insertions(+), 25 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 3e879d6a..5c31207a 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -124,12 +124,12 @@ where
 impl<T, const N: usize, Idx> Index<Idx> for CharWindow<T, N>
 where
     T: Iterator<Item = char>,
-    Idx: SliceIndex<[Option<char>], Output = Option<char>>,
+    Idx: SliceIndex<[Option<char>]>,
 {
-    type Output = Option<char>;
+    type Output = Idx::Output;
 
     fn index(&self, index: Idx) -> &Self::Output {
-        self.window.index(index)
+        &self.window[index]
     }
 }
 
@@ -199,12 +199,12 @@ where
     fn next(&mut self) -> Option<Self::Item> {
         // Collapse \r\n into \n
         loop {
-            match (self.window[0], self.window[1]) {
-                (Some('\r'), Some('\n')) => {
+            match self.window[..2] {
+                [Some('\r'), Some('\n')] => {
                     // Windows EOL into \n
                     self.shift();
                 }
-                (Some('\r'), _) => {
+                [Some('\r'), _] => {
                     // MAC EOL into \n
                     self.window.change_first('\n');
                 }
@@ -309,20 +309,20 @@ where
     /// Numeric lexing. The feast can start!
     fn lex_number(&mut self) -> LexResult {
         let start_pos = self.get_pos();
-        match (self.window[0], self.window[1]) {
-            (Some('0'), Some('x' | 'X')) => {
+        match self.window[..2] {
+            [Some('0'), Some('x' | 'X')] => {
                 // Hex! (0xdeadbeef)
                 self.next_char();
                 self.next_char();
                 self.lex_number_radix(start_pos, 16)
             }
-            (Some('0'), Some('o' | 'O')) => {
+            [Some('0'), Some('o' | 'O')] => {
                 // Octal style! (0o377)
                 self.next_char();
                 self.next_char();
                 self.lex_number_radix(start_pos, 8)
             }
-            (Some('0'), Some('b' | 'B')) => {
+            [Some('0'), Some('b' | 'B')] => {
                 // Binary! (0b_1110_0101)
                 self.next_char();
                 self.next_char();
@@ -470,9 +470,9 @@ where
 
     /// Test if we face '[eE][-+]?[0-9]+'
     fn at_exponent(&self) -> bool {
-        match (self.window[0], self.window[1]) {
-            (Some('e' | 'E'), Some('+' | '-')) => matches!(self.window[2], Some('0'..='9')),
-            (Some('e' | 'E'), Some('0'..='9')) => true,
+        match self.window[..2] {
+            [Some('e' | 'E'), Some('+' | '-')] => matches!(self.window[2], Some('0'..='9')),
+            [Some('e' | 'E'), Some('0'..='9')] => true,
             _ => false,
         }
     }
@@ -500,14 +500,13 @@ where
 
         // If the next two characters are also the quote character, then we have a triple-quoted
         // string; consume those two characters and ensure that we require a triple-quote to close
-        let triple_quoted =
-            if self.window[0] == Some(quote_char) && self.window[1] == Some(quote_char) {
-                self.next_char();
-                self.next_char();
-                true
-            } else {
-                false
-            };
+        let triple_quoted = if self.window[..2] == [Some(quote_char); 2] {
+            self.next_char();
+            self.next_char();
+            true
+        } else {
+            false
+        };
 
         loop {
             match self.next_char() {
@@ -534,9 +533,7 @@ where
                             // Look ahead at the next two characters; if we have two more
                             // quote_chars, it's the end of the string; consume the remaining
                             // closing quotes and break the loop
-                            if self.window[0] == Some(quote_char)
-                                && self.window[1] == Some(quote_char)
-                            {
+                            if self.window[..2] == [Some(quote_char); 2] {
                                 self.next_char();
                                 self.next_char();
                                 break;
@@ -1094,7 +1091,7 @@ where
                 } else {
                     let tok_start = self.get_pos();
                     self.next_char();
-                    if let (Some('.'), Some('.')) = (&self.window[0], &self.window[1]) {
+                    if let [Some('.'), Some('.')] = self.window[..2] {
                         self.next_char();
                         self.next_char();
                         let tok_end = self.get_pos();

From 674ab0d55e56089673d990c57c398c4a334d025c Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Tue, 3 Jan 2023 23:55:50 +0900
Subject: [PATCH 372/459] Remove incorrect EmptyExpression in
 parse_formatted_value

---
 parser/src/error.rs         | 2 +-
 parser/src/string_parser.rs | 6 +-----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/parser/src/error.rs b/parser/src/error.rs
index 7a4f60b2..eda73b57 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -42,7 +42,7 @@ impl fmt::Display for LexicalErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             LexicalErrorType::StringError => write!(f, "Got unexpected string"),
-            LexicalErrorType::FStringError(error) => write!(f, "Got error in f-string: {error}"),
+            LexicalErrorType::FStringError(error) => write!(f, "f-string: {error}"),
             LexicalErrorType::UnicodeError => write!(f, "Got unexpected unicode"),
             LexicalErrorType::NestingError => write!(f, "Got unexpected nesting"),
             LexicalErrorType::IndentationError => {
diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
index b3054935..df751e23 100644
--- a/parser/src/string_parser.rs
+++ b/parser/src/string_parser.rs
@@ -358,11 +358,7 @@ impl<'a> StringParser<'a> {
                 }
             }
         }
-        Err(if expression.trim().is_empty() {
-            FStringError::new(EmptyExpression, self.get_pos()).into()
-        } else {
-            FStringError::new(UnclosedLbrace, self.get_pos()).into()
-        })
+        Err(FStringError::new(UnclosedLbrace, self.get_pos()).into())
     }
 
     fn parse_spec(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {

From 0ca8d9ea4bfd0004b7b8551afb00ca787b1ee49f Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Wed, 4 Jan 2023 00:03:18 +0900
Subject: [PATCH 373/459] Address comment

---
 parser/src/lexer.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 5c31207a..939e8a4a 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -1091,7 +1091,7 @@ where
                 } else {
                     let tok_start = self.get_pos();
                     self.next_char();
-                    if let [Some('.'), Some('.')] = self.window[..2] {
+                    if self.window[..2] == [Some('.'); 2] {
                         self.next_char();
                         self.next_char();
                         let tok_end = self.get_pos();

From 72185fecd54d9215dc4ff8fc5a1838a83d0f941d Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Wed, 4 Jan 2023 13:16:03 +0900
Subject: [PATCH 374/459] Add with_col_offset and with_row_offset to Location

---
 codegen/src/symboltable.rs  |  2 +-
 core/src/location.rs        | 40 +++++++++++++++++++++++++++++++++++++
 parser/src/error.rs         |  2 +-
 parser/src/string_parser.rs |  8 ++------
 4 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 3182f3d3..a12ad5f9 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -168,7 +168,7 @@ impl SymbolTableError {
     pub fn into_codegen_error(self, source_path: String) -> CodegenError {
         CodegenError {
             error: CodegenErrorType::SyntaxError(self.error),
-            location: Location::new(self.location.row(), self.location.column() + 1),
+            location: self.location.with_col_offset(1),
             source_path,
         }
     }
diff --git a/core/src/location.rs b/core/src/location.rs
index 7f1adc44..b7b2c8b3 100644
--- a/core/src/location.rs
+++ b/core/src/location.rs
@@ -58,6 +58,34 @@ impl Location {
         self.row += 1;
         self.column = 0;
     }
+
+    pub fn with_col_offset<T: TryInto<isize>>(&self, offset: T) -> Self
+    where
+        <T as TryInto<isize>>::Error: std::fmt::Debug,
+    {
+        let column = (self.column as isize
+            + offset
+                .try_into()
+                .expect("offset should be able to convert to isize")) as u32;
+        Location {
+            row: self.row,
+            column,
+        }
+    }
+
+    pub fn with_row_offset<T: TryInto<isize>>(&self, offset: T) -> Self
+    where
+        <T as TryInto<isize>>::Error: std::fmt::Debug,
+    {
+        let row = (self.row as isize
+            + offset
+                .try_into()
+                .expect("offset should be able to convert to isize")) as u32;
+        Location {
+            row,
+            column: self.column,
+        }
+    }
 }
 
 #[cfg(test)]
@@ -77,4 +105,16 @@ mod tests {
         assert!(Location::new(1, 1) < Location::new(2, 1));
         assert!(Location::new(1, 2) < Location::new(2, 1));
     }
+
+    #[test]
+    fn test_with_col_offset() {
+        assert_eq!(Location::new(1, 1).with_col_offset(1), Location::new(1, 2));
+        assert_eq!(Location::new(1, 1).with_col_offset(-1), Location::new(1, 0));
+    }
+
+    #[test]
+    fn test_with_row_offset() {
+        assert_eq!(Location::new(1, 1).with_row_offset(1), Location::new(2, 1));
+        assert_eq!(Location::new(1, 1).with_row_offset(-1), Location::new(0, 1));
+    }
 }
diff --git a/parser/src/error.rs b/parser/src/error.rs
index 7a4f60b2..6df758be 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -213,7 +213,7 @@ pub(crate) fn parse_error_from_lalrpop(
             let expected = (expected.len() == 1).then(|| expected[0].clone());
             ParseError {
                 error: ParseErrorType::UnrecognizedToken(token.1, expected),
-                location: Location::new(token.0.row(), token.0.column() + 1),
+                location: token.0.with_col_offset(1),
                 source_path,
             }
         }
diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
index b3054935..52818bf0 100644
--- a/parser/src/string_parser.rs
+++ b/parser/src/string_parser.rs
@@ -32,7 +32,7 @@ impl<'a> StringParser<'a> {
             kind,
             str_start,
             str_end,
-            location: Location::new(str_start.row(), str_start.column() + offset),
+            location: str_start.with_col_offset(offset),
         }
     }
 
@@ -523,11 +523,7 @@ impl<'a> StringParser<'a> {
 
 fn parse_fstring_expr(source: &str, location: Location) -> Result<Expr, ParseError> {
     let fstring_body = format!("({source})");
-    parse_expression_located(
-        &fstring_body,
-        "<fstring>",
-        Location::new(location.row(), location.column() - 1),
-    )
+    parse_expression_located(&fstring_body, "<fstring>", location.with_col_offset(-1))
 }
 
 pub fn parse_string(

From 89ff74457eebafd9bbe02e1c9a0c05846f45f5b4 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Wed, 4 Jan 2023 13:19:23 +0900
Subject: [PATCH 375/459] Use Self

---
 core/src/location.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/location.rs b/core/src/location.rs
index b7b2c8b3..021dc2fd 100644
--- a/core/src/location.rs
+++ b/core/src/location.rs
@@ -67,7 +67,7 @@ impl Location {
             + offset
                 .try_into()
                 .expect("offset should be able to convert to isize")) as u32;
-        Location {
+        Self {
             row: self.row,
             column,
         }
@@ -81,7 +81,7 @@ impl Location {
             + offset
                 .try_into()
                 .expect("offset should be able to convert to isize")) as u32;
-        Location {
+        Self {
             row,
             column: self.column,
         }

From 7a78e12d5203726543b4850147dc2454d3b83ea7 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Wed, 4 Jan 2023 23:57:50 +0900
Subject: [PATCH 376/459] Simplify string check

---
 parser/src/lexer.rs | 65 +++++++++++----------------------------------
 parser/src/token.rs | 17 ++++++++++++
 2 files changed, 33 insertions(+), 49 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 939e8a4a..7ba9897e 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -241,59 +241,23 @@ where
 
     // Lexer helper functions:
     fn lex_identifier(&mut self) -> LexResult {
-        let mut name = String::new();
-        let start_pos = self.get_pos();
-
         // Detect potential string like rb'' b'' f'' u'' r''
-        let mut saw_b = false;
-        let mut saw_r = false;
-        let mut saw_u = false;
-        let mut saw_f = false;
-        loop {
-            // Detect r"", f"", b"" and u""
-            if !(saw_b || saw_u || saw_f) && matches!(self.window[0], Some('b' | 'B')) {
-                saw_b = true;
-            } else if !(saw_b || saw_r || saw_u || saw_f)
-                && matches!(self.window[0], Some('u' | 'U'))
-            {
-                saw_u = true;
-            } else if !(saw_r || saw_u) && matches!(self.window[0], Some('r' | 'R')) {
-                saw_r = true;
-            } else if !(saw_b || saw_u || saw_f) && matches!(self.window[0], Some('f' | 'F')) {
-                saw_f = true;
-            } else {
-                break;
+        match self.window[..3] {
+            [Some(c), Some('"' | '\''), ..] => {
+                if let Ok(kind) = c.to_string().try_into() {
+                    return self.lex_string(kind);
+                }
             }
-
-            // Take up char into name:
-            name.push(self.next_char().unwrap());
-
-            // Check if we have a string:
-            if matches!(self.window[0], Some('"' | '\'')) {
-                let kind = if saw_r {
-                    if saw_b {
-                        StringKind::RawBytes
-                    } else if saw_f {
-                        StringKind::RawFString
-                    } else {
-                        StringKind::RawString
-                    }
-                } else if saw_b {
-                    StringKind::Bytes
-                } else if saw_u {
-                    StringKind::Unicode
-                } else if saw_f {
-                    StringKind::FString
-                } else {
-                    StringKind::String
-                };
-
-                return self
-                    .lex_string(kind)
-                    .map(|(_, tok, end_pos)| (start_pos, tok, end_pos));
+            [Some(c1), Some(c2), Some('"' | '\'')] => {
+                if let Ok(kind) = format!("{c1}{c2}").try_into() {
+                    return self.lex_string(kind);
+                }
             }
-        }
+            _ => {}
+        };
 
+        let start_pos = self.get_pos();
+        let mut name = String::new();
         while self.is_identifier_continuation() {
             name.push(self.next_char().unwrap());
         }
@@ -495,6 +459,9 @@ where
 
     fn lex_string(&mut self, kind: StringKind) -> LexResult {
         let start_pos = self.get_pos();
+        for _ in 0..kind.to_string().len() {
+            self.next_char();
+        }
         let quote_char = self.next_char().unwrap();
         let mut string_content = String::new();
 
diff --git a/parser/src/token.rs b/parser/src/token.rs
index ce48410e..e915429b 100644
--- a/parser/src/token.rs
+++ b/parser/src/token.rs
@@ -240,6 +240,23 @@ pub enum StringKind {
     Unicode,
 }
 
+impl TryFrom<String> for StringKind {
+    type Error = String;
+
+    fn try_from(value: String) -> Result<Self, String> {
+        match value.as_str() {
+            "" => Ok(StringKind::String),
+            "r" | "R" => Ok(StringKind::RawString),
+            "u" | "U" => Ok(StringKind::Unicode),
+            "b" | "B" => Ok(StringKind::Bytes),
+            "f" | "F" => Ok(StringKind::FString),
+            "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" => Ok(StringKind::RawFString),
+            "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB" => Ok(StringKind::RawBytes),
+            s => Err(format!("Unexpected string prefix: {s}")),
+        }
+    }
+}
+
 impl fmt::Display for StringKind {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         use StringKind::*;

From 5373dcd74d6639750d53acc8122607355c1d1b00 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Thu, 5 Jan 2023 01:18:30 +0900
Subject: [PATCH 377/459] Use try_from

---
 parser/src/lexer.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 7ba9897e..d266f649 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -244,12 +244,12 @@ where
         // Detect potential string like rb'' b'' f'' u'' r''
         match self.window[..3] {
             [Some(c), Some('"' | '\''), ..] => {
-                if let Ok(kind) = c.to_string().try_into() {
+                if let Ok(kind) = StringKind::try_from(c.to_string()) {
                     return self.lex_string(kind);
                 }
             }
             [Some(c1), Some(c2), Some('"' | '\'')] => {
-                if let Ok(kind) = format!("{c1}{c2}").try_into() {
+                if let Ok(kind) = StringKind::try_from(format!("{c1}{c2}")) {
                     return self.lex_string(kind);
                 }
             }

From eb75c4fa4621591fbe042f73ed48a382b8747371 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Thu, 5 Jan 2023 18:24:54 +0900
Subject: [PATCH 378/459] Address comments

---
 parser/src/lexer.rs         |  6 +++---
 parser/src/string_parser.rs |  2 +-
 parser/src/token.rs         | 42 +++++++++++++++++++++++++++----------
 3 files changed, 35 insertions(+), 15 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index d266f649..82517945 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -244,12 +244,12 @@ where
         // Detect potential string like rb'' b'' f'' u'' r''
         match self.window[..3] {
             [Some(c), Some('"' | '\''), ..] => {
-                if let Ok(kind) = StringKind::try_from(c.to_string()) {
+                if let Ok(kind) = StringKind::try_from(c) {
                     return self.lex_string(kind);
                 }
             }
             [Some(c1), Some(c2), Some('"' | '\'')] => {
-                if let Ok(kind) = StringKind::try_from(format!("{c1}{c2}")) {
+                if let Ok(kind) = StringKind::try_from([c1, c2]) {
                     return self.lex_string(kind);
                 }
             }
@@ -459,7 +459,7 @@ where
 
     fn lex_string(&mut self, kind: StringKind) -> LexResult {
         let start_pos = self.get_pos();
-        for _ in 0..kind.to_string().len() {
+        for _ in 0..kind.prefix_len() {
             self.next_char();
         }
         let quote_char = self.next_char().unwrap();
diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
index 4e94347a..1d95cff8 100644
--- a/parser/src/string_parser.rs
+++ b/parser/src/string_parser.rs
@@ -26,7 +26,7 @@ impl<'a> StringParser<'a> {
         str_start: Location,
         str_end: Location,
     ) -> Self {
-        let offset = kind.to_string().len() + if triple_quoted { 3 } else { 1 };
+        let offset = kind.prefix_len() + if triple_quoted { 3 } else { 1 };
         Self {
             chars: source.chars().peekable(),
             kind,
diff --git a/parser/src/token.rs b/parser/src/token.rs
index e915429b..c6185391 100644
--- a/parser/src/token.rs
+++ b/parser/src/token.rs
@@ -240,19 +240,30 @@ pub enum StringKind {
     Unicode,
 }
 
-impl TryFrom<String> for StringKind {
+impl TryFrom<char> for StringKind {
     type Error = String;
 
-    fn try_from(value: String) -> Result<Self, String> {
-        match value.as_str() {
-            "" => Ok(StringKind::String),
-            "r" | "R" => Ok(StringKind::RawString),
-            "u" | "U" => Ok(StringKind::Unicode),
-            "b" | "B" => Ok(StringKind::Bytes),
-            "f" | "F" => Ok(StringKind::FString),
-            "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" => Ok(StringKind::RawFString),
-            "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB" => Ok(StringKind::RawBytes),
-            s => Err(format!("Unexpected string prefix: {s}")),
+    fn try_from(ch: char) -> Result<Self, String> {
+        match ch {
+            'r' | 'R' => Ok(StringKind::RawString),
+            'f' | 'F' => Ok(StringKind::FString),
+            'u' | 'U' => Ok(StringKind::Unicode),
+            'b' | 'B' => Ok(StringKind::Bytes),
+            c => Err(format!("Unexpected string prefix: {c}")),
+        }
+    }
+}
+
+impl TryFrom<[char; 2]> for StringKind {
+    type Error = String;
+
+    fn try_from(chars: [char; 2]) -> Result<Self, String> {
+        match chars {
+            ['r' | 'R', 'f' | 'F'] => Ok(StringKind::RawFString),
+            ['f' | 'F', 'r' | 'R'] => Ok(StringKind::RawFString),
+            ['r' | 'R', 'b' | 'B'] => Ok(StringKind::RawBytes),
+            ['b' | 'B', 'r' | 'R'] => Ok(StringKind::RawBytes),
+            [c1, c2] => Err(format!("Unexpected string prefix: {c1}{c2}")),
         }
     }
 }
@@ -291,4 +302,13 @@ impl StringKind {
     pub fn is_unicode(&self) -> bool {
         matches!(self, StringKind::Unicode)
     }
+
+    pub fn prefix_len(&self) -> usize {
+        use StringKind::*;
+        match self {
+            String => 0,
+            RawString | FString | Unicode | Bytes => 1,
+            RawFString | RawBytes => 2,
+        }
+    }
 }

From 60cbfd3a948af345d7c14ba34eb8b97fd823b986 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Thu, 5 Jan 2023 22:48:47 +0900
Subject: [PATCH 379/459] Implement Default for Location

---
 core/src/location.rs        | 8 +++++++-
 parser/src/lexer.rs         | 2 +-
 parser/src/string_parser.rs | 4 ++--
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/core/src/location.rs b/core/src/location.rs
index 021dc2fd..9d32b54a 100644
--- a/core/src/location.rs
+++ b/core/src/location.rs
@@ -1,12 +1,18 @@
 use serde::{Deserialize, Serialize};
 
 /// Sourcecode location.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
 pub struct Location {
     pub(super) row: u32,
     pub(super) column: u32,
 }
 
+impl Default for Location {
+    fn default() -> Self {
+        Self { row: 1, column: 0 }
+    }
+}
+
 impl Location {
     pub fn fmt_with(
         &self,
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 82517945..ec4cfbed 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -153,7 +153,7 @@ pub type LexResult = Result<Spanned, LexicalError>;
 
 #[inline]
 pub fn make_tokenizer(source: &str) -> impl Iterator<Item = LexResult> + '_ {
-    make_tokenizer_located(source, Location::new(1, 0))
+    make_tokenizer_located(source, Location::default())
 }
 
 pub fn make_tokenizer_located(
diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
index 1d95cff8..c18ba0f9 100644
--- a/parser/src/string_parser.rs
+++ b/parser/src/string_parser.rs
@@ -541,8 +541,8 @@ mod tests {
             source,
             StringKind::FString,
             false,
-            Location::new(1, 0),
-            Location::new(1, source.len() + 3), // 3 for prefix and quotes
+            Location::default(),
+            Location::default().with_col_offset(source.len() + 3), // 3 for prefix and quotes
         )
         .parse()
     }

From 1a28e49adc99d2df099cc6f068c43ef548801c37 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Fri, 6 Jan 2023 23:29:20 +0900
Subject: [PATCH 380/459] Include comment text in token

---
 parser/python.lalrpop |  2 +-
 parser/src/lexer.rs   | 11 ++++++-----
 parser/src/parser.rs  |  4 ++--
 parser/src/token.rs   |  4 ++--
 4 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 9e6dd988..90f4ce25 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -1463,6 +1463,6 @@ extern {
         name => lexer::Tok::Name { name: <String> },
         "\n" => lexer::Tok::Newline,
         ";" => lexer::Tok::Semi,
-        "#" => lexer::Tok::Comment,
+        "#" => lexer::Tok::Comment(_),
     }
 }
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index ec4cfbed..cb38503b 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -444,16 +444,17 @@ where
     /// Skip everything until end of line
     fn lex_comment(&mut self) -> LexResult {
         let start_pos = self.get_pos();
-        self.next_char();
+        let mut value = String::new();
+        value.push(self.next_char().unwrap());
         loop {
             match self.window[0] {
                 Some('\n') | None => {
                     let end_pos = self.get_pos();
-                    return Ok((start_pos, Tok::Comment, end_pos));
+                    return Ok((start_pos, Tok::Comment(value), end_pos));
                 }
                 Some(_) => {}
             }
-            self.next_char();
+            value.push(self.next_char().unwrap());
         }
     }
 
@@ -1266,7 +1267,7 @@ mod tests {
             fn $name() {
                 let source = format!(r"99232  # {}", $eol);
                 let tokens = lex_source(&source);
-                assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Comment, Tok::Newline]);
+                assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Comment(format!("# {}", $eol)), Tok::Newline]);
             }
             )*
         }
@@ -1290,7 +1291,7 @@ mod tests {
                     tokens,
                     vec![
                         Tok::Int { value: BigInt::from(123) },
-                        Tok::Comment,
+                        Tok::Comment("# Foo".to_string()),
                         Tok::Newline,
                         Tok::Int { value: BigInt::from(456) },
                         Tok::Newline,
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index aad02882..4277d300 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -96,7 +96,7 @@ pub fn parse_located(
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
     let tokenizer = iter::once(Ok(marker_token))
         .chain(lxr)
-        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment));
+        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. }));
 
     python::TopParser::new()
         .parse(tokenizer)
@@ -112,7 +112,7 @@ pub fn parse_tokens(
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
     let tokenizer = iter::once(Ok(marker_token))
         .chain(lxr)
-        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment));
+        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment(_)));
 
     python::TopParser::new()
         .parse(tokenizer)
diff --git a/parser/src/token.rs b/parser/src/token.rs
index c6185391..72aa89d6 100644
--- a/parser/src/token.rs
+++ b/parser/src/token.rs
@@ -37,7 +37,7 @@ pub enum Tok {
     Rsqb,
     Colon,
     Comma,
-    Comment,
+    Comment(String),
     Semi,
     Plus,
     Minus,
@@ -148,7 +148,7 @@ impl fmt::Display for Tok {
             Rsqb => f.write_str("']'"),
             Colon => f.write_str("':'"),
             Comma => f.write_str("','"),
-            Comment => f.write_str("#"),
+            Comment(value) => f.write_str(value),
             Semi => f.write_str("';'"),
             Plus => f.write_str("'+'"),
             Minus => f.write_str("'-'"),

From 9ff117caa000272959d08902b9448c9864dab973 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Sat, 7 Jan 2023 20:14:24 +0900
Subject: [PATCH 381/459] Fix nightly clippy warnings

---
 ast/src/constant.rs | 6 +++---
 ast/src/unparse.rs  | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ast/src/constant.rs b/ast/src/constant.rs
index 373f5f97..adec47a1 100644
--- a/ast/src/constant.rs
+++ b/ast/src/constant.rs
@@ -46,7 +46,7 @@ impl std::fmt::Display for Constant {
             Constant::Int(i) => i.fmt(f),
             Constant::Tuple(tup) => {
                 if let [elt] = &**tup {
-                    write!(f, "({},)", elt)
+                    write!(f, "({elt},)")
                 } else {
                     f.write_str("(")?;
                     for (i, elt) in tup.iter().enumerate() {
@@ -61,9 +61,9 @@ impl std::fmt::Display for Constant {
             Constant::Float(fp) => f.pad(&rustpython_common::float_ops::to_string(*fp)),
             Constant::Complex { real, imag } => {
                 if *real == 0.0 {
-                    write!(f, "{}j", imag)
+                    write!(f, "{imag}j")
                 } else {
-                    write!(f, "({}{:+}j)", real, imag)
+                    write!(f, "({real}{imag:+}j)")
                 }
             }
             Constant::Ellipsis => f.pad("..."),
diff --git a/ast/src/unparse.rs b/ast/src/unparse.rs
index 70addae9..551283c0 100644
--- a/ast/src/unparse.rs
+++ b/ast/src/unparse.rs
@@ -400,7 +400,7 @@ impl<'a> Unparser<'a> {
                 .checked_sub(defaults_start)
                 .and_then(|i| args.kw_defaults.get(i))
             {
-                write!(self, "={}", default)?;
+                write!(self, "={default}")?;
             }
         }
         if let Some(kwarg) = &args.kwarg {

From 2f33de77c4cd6befa44089ffcaf873a297e71d84 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 9 Jan 2023 19:35:06 +0900
Subject: [PATCH 382/459] first cspell dict

---
 parser/src/function.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/parser/src/function.rs b/parser/src/function.rs
index bb8feeab..21ccf013 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -43,7 +43,7 @@ pub fn validate_arguments(arguments: ast::Arguments) -> Result<ast::Arguments, L
 pub fn parse_params(
     params: (Vec<ParameterDef>, Vec<ParameterDef>),
 ) -> Result<ParameterDefs, LexicalError> {
-    let mut posonly = Vec::with_capacity(params.0.len());
+    let mut pos_only = Vec::with_capacity(params.0.len());
     let mut names = Vec::with_capacity(params.1.len());
     let mut defaults = vec![];
 
@@ -63,7 +63,7 @@ pub fn parse_params(
 
     for (name, default) in params.0 {
         try_default(&name, default)?;
-        posonly.push(name);
+        pos_only.push(name);
     }
 
     for (name, default) in params.1 {
@@ -71,7 +71,7 @@ pub fn parse_params(
         names.push(name);
     }
 
-    Ok((posonly, names, defaults))
+    Ok((pos_only, names, defaults))
 }
 
 type FunctionArgument = (

From e3fa5a54badebe56e8ab246c95dfb3a6a8749d36 Mon Sep 17 00:00:00 2001
From: Noa <coolreader18@gmail.com>
Date: Wed, 11 Jan 2023 00:05:25 -0600
Subject: [PATCH 383/459] Bump all crate versions to 0.2.0

---
 Cargo.toml         | 2 +-
 ast/Cargo.toml     | 9 ++++++---
 codegen/Cargo.toml | 4 ++--
 core/Cargo.toml    | 2 +-
 parser/Cargo.toml  | 6 +++---
 5 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 118286f4..34dbad32 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "rustpython-compiler"
-version = "0.1.2"
+version = "0.2.0"
 description = "A usability wrapper around rustpython-parser and rustpython-compiler-core"
 authors = ["RustPython Team"]
 edition = "2021"
diff --git a/ast/Cargo.toml b/ast/Cargo.toml
index 7efefb3a..08f29fcc 100644
--- a/ast/Cargo.toml
+++ b/ast/Cargo.toml
@@ -1,8 +1,11 @@
 [package]
 name = "rustpython-ast"
-version = "0.1.0"
+version = "0.2.0"
+description = "AST definitions for RustPython"
 authors = ["RustPython Team"]
 edition = "2021"
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
 
 [features]
 default = ["constant-optimization", "fold"]
@@ -12,5 +15,5 @@ unparse = ["rustpython-common"]
 
 [dependencies]
 num-bigint = "0.4.3"
-rustpython-compiler-core = { path = "../core" }
-rustpython-common = { path = "../../common", optional = true }
+rustpython-compiler-core = { path = "../core", version = "0.2.0" }
+rustpython-common = { path = "../../common", version = "0.2.0", optional = true }
diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml
index 253d976c..4265e4b2 100644
--- a/codegen/Cargo.toml
+++ b/codegen/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "rustpython-codegen"
-version = "0.1.2"
+version = "0.2.0"
 description = "Compiler for python code into bytecode for the rustpython VM."
 authors = ["RustPython Team"]
 repository = "https://github.com/RustPython/RustPython"
@@ -9,7 +9,7 @@ edition = "2021"
 
 [dependencies]
 rustpython-ast = { path = "../ast", features = ["unparse"] }
-rustpython-compiler-core = { path = "../core", version = "0.1.1" }
+rustpython-compiler-core = { path = "../core", version = "0.2.0" }
 
 ahash = "0.7.6"
 bitflags = "1.3.2"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 65be384f..a3dd82bb 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "rustpython-compiler-core"
 description = "RustPython specific bytecode."
-version = "0.1.2"
+version = "0.2.0"
 authors = ["RustPython Team"]
 edition = "2021"
 repository = "https://github.com/RustPython/RustPython"
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index e3ce5a39..472e309b 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "rustpython-parser"
-version = "0.1.2"
+version = "0.2.0"
 description = "Parser for python code."
 authors = ["RustPython Team"]
 build = "build.rs"
@@ -18,8 +18,8 @@ phf_codegen = "0.10"
 tiny-keccak = { version = "2", features = ["sha3"] }
 
 [dependencies]
-rustpython-ast = { path = "../ast" }
-rustpython-compiler-core = { path = "../core" }
+rustpython-ast = { path = "../ast", version = "0.2.0" }
+rustpython-compiler-core = { path = "../core", version = "0.2.0" }
 
 ahash = "0.7.6"
 itertools = "0.10.3"

From fea57bcb04c4f1b5237aa2f761d7460914973299 Mon Sep 17 00:00:00 2001
From: Martin Fischer <martin@push-f.com>
Date: Wed, 11 Jan 2023 09:11:12 +0100
Subject: [PATCH 384/459] Fix docs.rs build for rustpython-parser

docs.rs failed to build the documentation of the recently released
rustpython-parser 0.2.0 because the build.rs script couldn't write the
parser.rs file because docs.rs builds the documentation in a sandbox
with a read-only filesystem.

This commit fixes this by writing the parser.rs file to the cargo output
directory instead, as recommended by the docs.rs documentation.[1]

Fixes #4436.

[1]: https://docs.rs/about/builds#read-only-directories
---
 parser/build.rs      | 29 +++++++++++++++++------------
 parser/src/python.rs |  2 +-
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/parser/build.rs b/parser/build.rs
index 4559f263..f9a3439b 100644
--- a/parser/build.rs
+++ b/parser/build.rs
@@ -1,22 +1,22 @@
 use std::fmt::Write as _;
 use std::fs::File;
 use std::io::{BufRead, BufReader, BufWriter, Write};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use tiny_keccak::{Hasher, Sha3};
 
 fn main() -> anyhow::Result<()> {
     const SOURCE: &str = "python.lalrpop";
-    const TARGET: &str = "python.rs";
+    let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
 
     println!("cargo:rerun-if-changed={SOURCE}");
 
-    try_lalrpop(SOURCE, TARGET)?;
-    gen_phf();
+    try_lalrpop(SOURCE, &out_dir.join("python.rs"))?;
+    gen_phf(&out_dir);
 
     Ok(())
 }
 
-fn requires_lalrpop(source: &str, target: &str) -> Option<String> {
+fn requires_lalrpop(source: &str, target: &Path) -> Option<String> {
     let Ok(target) = File::open(target) else {
         return Some("python.rs doesn't exist. regenerate.".to_owned());
     };
@@ -68,16 +68,22 @@ fn requires_lalrpop(source: &str, target: &str) -> Option<String> {
     None
 }
 
-fn try_lalrpop(source: &str, target: &str) -> anyhow::Result<()> {
+fn try_lalrpop(source: &str, target: &Path) -> anyhow::Result<()> {
     let Some(_message) = requires_lalrpop(source, target) else {
         return Ok(());
     };
 
     #[cfg(feature = "lalrpop")]
-    lalrpop::process_root().unwrap_or_else(|e| {
-        println!("cargo:warning={_message}");
-        panic!("running lalrpop failed. {e:?}");
-    });
+    // We are not using lalrpop::process_root() or Configuration::process_current_dir()
+    // because of https://github.com/lalrpop/lalrpop/issues/699.
+    lalrpop::Configuration::new()
+        .use_cargo_dir_conventions()
+        .set_in_dir(Path::new("."))
+        .process()
+        .unwrap_or_else(|e| {
+            println!("cargo:warning={_message}");
+            panic!("running lalrpop failed. {e:?}");
+        });
 
     #[cfg(not(feature = "lalrpop"))]
     {
@@ -98,8 +104,7 @@ fn sha_equal(expected_sha3_str: &str, actual_sha3: &[u8; 32]) -> bool {
     *actual_sha3 == expected_sha3
 }
 
-fn gen_phf() {
-    let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
+fn gen_phf(out_dir: &Path) {
     let mut kwds = phf_codegen::Map::new();
     let kwds = kwds
         // Alphabetical keywords:
diff --git a/parser/src/python.rs b/parser/src/python.rs
index e38af9b7..a00274d6 100644
--- a/parser/src/python.rs
+++ b/parser/src/python.rs
@@ -1,3 +1,3 @@
 #![allow(clippy::all)]
 #![allow(unused)]
-include!("../python.rs");
+include!(concat!(env!("OUT_DIR"), "/python.rs"));

From 5cc208cc433f13fe48d364b8b849ee14020e634f Mon Sep 17 00:00:00 2001
From: Ran Benita <ran@unusedvar.com>
Date: Thu, 12 Jan 2023 14:53:55 +0200
Subject: [PATCH 385/459] Add NonLogicalNewline token

This token is completely ignored by the parser, but it's useful for
other users of the lexer, such as the Ruff linter. For example, the
token is helpful for a "trailing comma" lint.

The same idea exists in Python's `tokenize` module - there is a NEWLINE
token (logical newline), and a NL token (non-logical newline).

Fixes #4385.
---
 parser/src/lexer.rs  | 83 ++++++++++++++++++++++++++++++++++++++++++--
 parser/src/parser.rs |  2 +-
 parser/src/token.rs  |  2 ++
 3 files changed, 84 insertions(+), 3 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index cb38503b..1c124f62 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -1075,10 +1075,13 @@ where
                 self.next_char();
                 let tok_end = self.get_pos();
 
-                // Depending on the nesting level, we emit newline or not:
+                // Depending on the nesting level, we emit a logical or
+                // non-logical newline:
                 if self.nesting == 0 {
                     self.at_begin_of_line = true;
                     self.emit((tok_start, Tok::Newline, tok_end));
+                } else {
+                    self.emit((tok_start, Tok::NonLogicalNewline, tok_end));
                 }
             }
             ' ' | '\t' | '\x0C' => {
@@ -1464,7 +1467,16 @@ mod tests {
         $(
             #[test]
             fn $name() {
-                let source = format!("x = [{}    1,2{}]{}", $eol, $eol, $eol);
+                let source = r"x = [
+
+    1,2
+,(3,
+4,
+), {
+5,
+6,\
+7}]
+".replace("\n", $eol);
                 let tokens = lex_source(&source);
                 assert_eq!(
                     tokens,
@@ -1474,9 +1486,32 @@ mod tests {
                         },
                         Tok::Equal,
                         Tok::Lsqb,
+                        Tok::NonLogicalNewline,
+                        Tok::NonLogicalNewline,
                         Tok::Int { value: BigInt::from(1) },
                         Tok::Comma,
                         Tok::Int { value: BigInt::from(2) },
+                        Tok::NonLogicalNewline,
+                        Tok::Comma,
+                        Tok::Lpar,
+                        Tok::Int { value: BigInt::from(3) },
+                        Tok::Comma,
+                        Tok::NonLogicalNewline,
+                        Tok::Int { value: BigInt::from(4) },
+                        Tok::Comma,
+                        Tok::NonLogicalNewline,
+                        Tok::Rpar,
+                        Tok::Comma,
+                        Tok::Lbrace,
+                        Tok::NonLogicalNewline,
+                        Tok::Int { value: BigInt::from(5) },
+                        Tok::Comma,
+                        Tok::NonLogicalNewline,
+                        Tok::Int { value: BigInt::from(6) },
+                        Tok::Comma,
+                        // Continuation here - no NonLogicalNewline.
+                        Tok::Int { value: BigInt::from(7) },
+                        Tok::Rbrace,
                         Tok::Rsqb,
                         Tok::Newline,
                     ]
@@ -1492,6 +1527,50 @@ mod tests {
         test_newline_in_brackets_unix_eol: UNIX_EOL,
     }
 
+    #[test]
+    fn test_non_logical_newline_in_string_continuation() {
+        let source = r"(
+    'a'
+    'b'
+
+    'c' \
+    'd'
+)";
+        let tokens = lex_source(source);
+        assert_eq!(
+            tokens,
+            vec![
+                Tok::Lpar,
+                Tok::NonLogicalNewline,
+                stok("a"),
+                Tok::NonLogicalNewline,
+                stok("b"),
+                Tok::NonLogicalNewline,
+                Tok::NonLogicalNewline,
+                stok("c"),
+                stok("d"),
+                Tok::NonLogicalNewline,
+                Tok::Rpar,
+                Tok::Newline,
+            ]
+        );
+    }
+
+    #[test]
+    fn test_logical_newline_line_comment() {
+        let source = "#Hello\n#World";
+        let tokens = lex_source(source);
+        assert_eq!(
+            tokens,
+            vec![
+                Tok::Comment("#Hello".to_owned()),
+                // tokenize.py does put an NL here...
+                Tok::Comment("#World".to_owned()),
+                // ... and here, but doesn't seem very useful.
+            ]
+        );
+    }
+
     #[test]
     fn test_operators() {
         let source = "//////=/ /";
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 4277d300..4d9b52bb 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -96,7 +96,7 @@ pub fn parse_located(
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
     let tokenizer = iter::once(Ok(marker_token))
         .chain(lxr)
-        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. }));
+        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
 
     python::TopParser::new()
         .parse(tokenizer)
diff --git a/parser/src/token.rs b/parser/src/token.rs
index 72aa89d6..b51b2f47 100644
--- a/parser/src/token.rs
+++ b/parser/src/token.rs
@@ -25,6 +25,7 @@ pub enum Tok {
         triple_quoted: bool,
     },
     Newline,
+    NonLogicalNewline,
     Indent,
     Dedent,
     StartModule,
@@ -136,6 +137,7 @@ impl fmt::Display for Tok {
                 write!(f, "{kind}{quotes}{value}{quotes}")
             }
             Newline => f.write_str("Newline"),
+            NonLogicalNewline => f.write_str("NonLogicalNewline"),
             Indent => f.write_str("Indent"),
             Dedent => f.write_str("Dedent"),
             StartModule => f.write_str("StartProgram"),

From da96cecfcaf202611e112ab1e278739cffaaa7a1 Mon Sep 17 00:00:00 2001
From: Noa <coolreader18@gmail.com>
Date: Fri, 9 Dec 2022 20:32:48 -0600
Subject: [PATCH 386/459] Switch from 64-bit instruction enum to out-of-line
 arg values

---
 codegen/src/compile.rs                        | 759 ++++++++++--------
 codegen/src/ir.rs                             | 192 +++--
 ...pile__tests__nested_double_async_with.snap |  20 +-
 core/Cargo.toml                               |   2 +-
 core/src/bytecode.rs                          | 698 +++++++++++-----
 src/lib.rs                                    |   5 +-
 6 files changed, 1062 insertions(+), 614 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index ba3d5453..2da3aba6 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -15,7 +15,10 @@ use itertools::Itertools;
 use num_complex::Complex64;
 use num_traits::ToPrimitive;
 use rustpython_ast as ast;
-use rustpython_compiler_core::{self as bytecode, CodeObject, ConstantData, Instruction, Location};
+use rustpython_compiler_core::{
+    self as bytecode, Arg as OpArgMarker, CodeObject, ConstantData, Instruction, Location, NameIdx,
+    OpArg, OpArgType,
+};
 use std::borrow::Cow;
 
 pub use rustpython_compiler_core::Mode;
@@ -35,23 +38,6 @@ enum CallType {
     Ex { has_kwargs: bool },
 }
 
-impl CallType {
-    fn normal_call(self) -> Instruction {
-        match self {
-            CallType::Positional { nargs } => Instruction::CallFunctionPositional { nargs },
-            CallType::Keyword { nargs } => Instruction::CallFunctionKeyword { nargs },
-            CallType::Ex { has_kwargs } => Instruction::CallFunctionEx { has_kwargs },
-        }
-    }
-    fn method_call(self) -> Instruction {
-        match self {
-            CallType::Positional { nargs } => Instruction::CallMethodPositional { nargs },
-            CallType::Keyword { nargs } => Instruction::CallMethodKeyword { nargs },
-            CallType::Ex { has_kwargs } => Instruction::CallMethodEx { has_kwargs },
-        }
-    }
-}
-
 fn is_forbidden_name(name: &str) -> bool {
     // See https://docs.python.org/3/library/constants.html#built-in-constants
     const BUILTIN_CONSTANTS: &[&str] = &["__debug__"];
@@ -197,6 +183,21 @@ pub fn compile_expression(
     )
 }
 
+macro_rules! emit {
+    ($c:expr, Instruction::$op:ident { $arg:ident$(,)? }$(,)?) => {
+        $c.emit_arg($arg, |x| Instruction::$op { $arg: x })
+    };
+    ($c:expr, Instruction::$op:ident { $arg:ident : $argval:expr $(,)? }$(,)?) => {
+        $c.emit_arg($argval, |x| Instruction::$op { $arg: x })
+    };
+    ($c:expr, Instruction::$op:ident( $argval:expr $(,)? )$(,)?) => {
+        $c.emit_arg($argval, Instruction::$op)
+    };
+    ($c:expr, Instruction::$op:ident$(,)?) => {
+        $c.emit_noarg(Instruction::$op)
+    };
+}
+
 impl Compiler {
     fn new(opts: CompileOpts, source_path: String, code_name: String) -> Self {
         let module_code = ir::CodeInfo {
@@ -209,7 +210,7 @@ impl Compiler {
             obj_name: code_name,
 
             blocks: vec![ir::Block::default()],
-            current_block: bytecode::Label(0),
+            current_block: ir::BlockIdx(0),
             constants: IndexSet::default(),
             name_cache: IndexSet::default(),
             varname_cache: IndexSet::default(),
@@ -290,7 +291,7 @@ impl Compiler {
             obj_name,
 
             blocks: vec![ir::Block::default()],
-            current_block: bytecode::Label(0),
+            current_block: ir::BlockIdx(0),
             constants: IndexSet::default(),
             name_cache: IndexSet::default(),
             varname_cache: IndexSet::default(),
@@ -341,11 +342,11 @@ impl Compiler {
         if let Some(value) = doc {
             self.emit_constant(ConstantData::Str { value });
             let doc = self.name("__doc__");
-            self.emit(Instruction::StoreGlobal(doc))
+            emit!(self, Instruction::StoreGlobal(doc))
         }
 
         if Self::find_ann(statements) {
-            self.emit(Instruction::SetupAnnotation);
+            emit!(self, Instruction::SetupAnnotation);
         }
 
         self.compile_statements(statements)?;
@@ -354,7 +355,7 @@ impl Compiler {
 
         // Emit None at end:
         self.emit_constant(ConstantData::None);
-        self.emit(Instruction::ReturnValue);
+        emit!(self, Instruction::ReturnValue);
         Ok(())
     }
 
@@ -369,7 +370,7 @@ impl Compiler {
             for statement in body {
                 if let ast::StmtKind::Expr { value } = &statement.node {
                     self.compile_expression(value)?;
-                    self.emit(Instruction::PrintExpr);
+                    emit!(self, Instruction::PrintExpr);
                 } else {
                     self.compile_statement(statement)?;
                 }
@@ -377,8 +378,8 @@ impl Compiler {
 
             if let ast::StmtKind::Expr { value } = &last.node {
                 self.compile_expression(value)?;
-                self.emit(Instruction::Duplicate);
-                self.emit(Instruction::PrintExpr);
+                emit!(self, Instruction::Duplicate);
+                emit!(self, Instruction::PrintExpr);
             } else {
                 self.compile_statement(last)?;
                 self.emit_constant(ConstantData::None);
@@ -387,7 +388,7 @@ impl Compiler {
             self.emit_constant(ConstantData::None);
         };
 
-        self.emit(Instruction::ReturnValue);
+        emit!(self, Instruction::ReturnValue);
         Ok(())
     }
 
@@ -409,13 +410,13 @@ impl Compiler {
                 | ast::StmtKind::AsyncFunctionDef { .. }
                 | ast::StmtKind::ClassDef { .. } => {
                     let store_inst = self.current_block().instructions.pop().unwrap(); // pop Instruction::Store
-                    self.emit(Instruction::Duplicate);
+                    emit!(self, Instruction::Duplicate);
                     self.current_block().instructions.push(store_inst);
                 }
                 _ => self.emit_constant(ConstantData::None),
             }
         }
-        self.emit(Instruction::ReturnValue);
+        emit!(self, Instruction::ReturnValue);
 
         Ok(())
     }
@@ -428,7 +429,7 @@ impl Compiler {
     ) -> CompileResult<()> {
         self.symbol_table_stack.push(symbol_table);
         self.compile_expression(expression)?;
-        self.emit(Instruction::ReturnValue);
+        emit!(self, Instruction::ReturnValue);
         Ok(())
     }
 
@@ -530,7 +531,7 @@ impl Compiler {
                 NameUsage::Delete => Instruction::DeleteLocal,
             },
         };
-        self.emit(op(idx as u32));
+        self.emit_arg(idx as NameIdx, op);
 
         Ok(())
     }
@@ -560,11 +561,11 @@ impl Compiler {
                     });
                     self.emit_constant(ConstantData::None);
                     let idx = self.name(&name.name);
-                    self.emit(Instruction::ImportName { idx });
+                    emit!(self, Instruction::ImportName { idx });
                     if let Some(alias) = &name.asname {
                         for part in name.name.split('.').skip(1) {
                             let idx = self.name(part);
-                            self.emit(Instruction::LoadAttr { idx });
+                            emit!(self, Instruction::LoadAttr { idx });
                         }
                         self.store_name(alias)?
                     } else {
@@ -606,14 +607,14 @@ impl Compiler {
                     elements: from_list,
                 });
                 if let Some(idx) = module_idx {
-                    self.emit(Instruction::ImportName { idx });
+                    emit!(self, Instruction::ImportName { idx });
                 } else {
-                    self.emit(Instruction::ImportNameless);
+                    emit!(self, Instruction::ImportNameless);
                 }
 
                 if import_star {
                     // from .... import *
-                    self.emit(Instruction::ImportStar);
+                    emit!(self, Instruction::ImportStar);
                 } else {
                     // from mod import a, b as c
 
@@ -621,7 +622,7 @@ impl Compiler {
                         let name = &name.node;
                         let idx = self.name(&name.name);
                         // import symbol from module:
-                        self.emit(Instruction::ImportFrom { idx });
+                        emit!(self, Instruction::ImportFrom { idx });
 
                         // Store module under proper name:
                         if let Some(alias) = &name.asname {
@@ -632,14 +633,14 @@ impl Compiler {
                     }
 
                     // Pop module from stack:
-                    self.emit(Instruction::Pop);
+                    emit!(self, Instruction::Pop);
                 }
             }
             Expr { value } => {
                 self.compile_expression(value)?;
 
                 // Pop result of stack, since we not use it:
-                self.emit(Instruction::Pop);
+                emit!(self, Instruction::Pop);
             }
             Global { .. } | Nonlocal { .. } => {
                 // Handled during symbol table construction.
@@ -655,9 +656,12 @@ impl Compiler {
                     let else_block = self.new_block();
                     self.compile_jump_if(test, false, else_block)?;
                     self.compile_statements(body)?;
-                    self.emit(Instruction::Jump {
-                        target: after_block,
-                    });
+                    emit!(
+                        self,
+                        Instruction::Jump {
+                            target: after_block,
+                        }
+                    );
 
                     // else:
                     self.switch_to_block(else_block);
@@ -697,7 +701,7 @@ impl Compiler {
                     }
                     None => bytecode::RaiseKind::Reraise,
                 };
-                self.emit(Instruction::Raise { kind });
+                emit!(self, Instruction::Raise { kind });
             }
             Try {
                 body,
@@ -749,26 +753,29 @@ impl Compiler {
                     self.compile_jump_if(test, true, after_block)?;
 
                     let assertion_error = self.name("AssertionError");
-                    self.emit(Instruction::LoadGlobal(assertion_error));
+                    emit!(self, Instruction::LoadGlobal(assertion_error));
                     match msg {
                         Some(e) => {
                             self.compile_expression(e)?;
-                            self.emit(Instruction::CallFunctionPositional { nargs: 1 });
+                            emit!(self, Instruction::CallFunctionPositional { nargs: 1 });
                         }
                         None => {
-                            self.emit(Instruction::CallFunctionPositional { nargs: 0 });
+                            emit!(self, Instruction::CallFunctionPositional { nargs: 0 });
                         }
                     }
-                    self.emit(Instruction::Raise {
-                        kind: bytecode::RaiseKind::Raise,
-                    });
+                    emit!(
+                        self,
+                        Instruction::Raise {
+                            kind: bytecode::RaiseKind::Raise,
+                        }
+                    );
 
                     self.switch_to_block(after_block);
                 }
             }
             Break => match self.ctx.loop_data {
                 Some((_, end)) => {
-                    self.emit(Instruction::Break { target: end });
+                    emit!(self, Instruction::Break { target: end });
                 }
                 None => {
                     return Err(self.error_loc(CodegenErrorType::InvalidBreak, statement.location));
@@ -776,7 +783,7 @@ impl Compiler {
             },
             Continue => match self.ctx.loop_data {
                 Some((start, _)) => {
-                    self.emit(Instruction::Continue { target: start });
+                    emit!(self, Instruction::Continue { target: start });
                 }
                 None => {
                     return Err(
@@ -808,14 +815,14 @@ impl Compiler {
                     }
                 }
 
-                self.emit(Instruction::ReturnValue);
+                emit!(self, Instruction::ReturnValue);
             }
             Assign { targets, value, .. } => {
                 self.compile_expression(value)?;
 
                 for (i, target) in targets.iter().enumerate() {
                     if i + 1 != targets.len() {
-                        self.emit(Instruction::Duplicate);
+                        emit!(self, Instruction::Duplicate);
                     }
                     self.compile_store(target)?;
                 }
@@ -846,12 +853,12 @@ impl Compiler {
                 self.check_forbidden_name(attr, NameUsage::Delete)?;
                 self.compile_expression(value)?;
                 let idx = self.name(attr);
-                self.emit(Instruction::DeleteAttr { idx });
+                emit!(self, Instruction::DeleteAttr { idx });
             }
             ast::ExprKind::Subscript { value, slice, .. } => {
                 self.compile_expression(value)?;
                 self.compile_expression(slice)?;
-                self.emit(Instruction::DeleteSubscript);
+                emit!(self, Instruction::DeleteSubscript);
             }
             ast::ExprKind::Tuple { elts, .. } | ast::ExprKind::List { elts, .. } => {
                 for element in elts {
@@ -878,10 +885,7 @@ impl Compiler {
             for element in &args.defaults {
                 self.compile_expression(element)?;
             }
-            self.emit(Instruction::BuildTuple {
-                size,
-                unpack: false,
-            });
+            emit!(self, Instruction::BuildTuple { size });
         }
 
         if !args.kw_defaults.is_empty() {
@@ -895,11 +899,12 @@ impl Compiler {
                 });
                 self.compile_expression(default)?;
             }
-            self.emit(Instruction::BuildMap {
-                size: args.kw_defaults.len() as u32,
-                unpack: false,
-                for_call: false,
-            });
+            emit!(
+                self,
+                Instruction::BuildMap {
+                    size: args.kw_defaults.len() as u32,
+                }
+            );
         }
 
         let mut funcflags = bytecode::MakeFunctionFlags::empty();
@@ -955,7 +960,7 @@ impl Compiler {
     fn apply_decorators(&mut self, decorator_list: &[ast::Expr]) {
         // Apply decorators:
         for _ in decorator_list {
-            self.emit(Instruction::CallFunctionPositional { nargs: 1 });
+            emit!(self, Instruction::CallFunctionPositional { nargs: 1 });
         }
     }
 
@@ -971,20 +976,26 @@ impl Compiler {
 
         // Setup a finally block if we have a finally statement.
         if !finalbody.is_empty() {
-            self.emit(Instruction::SetupFinally {
-                handler: finally_block,
-            });
+            emit!(
+                self,
+                Instruction::SetupFinally {
+                    handler: finally_block,
+                }
+            );
         }
 
         let else_block = self.new_block();
 
         // try:
-        self.emit(Instruction::SetupExcept {
-            handler: handler_block,
-        });
+        emit!(
+            self,
+            Instruction::SetupExcept {
+                handler: handler_block,
+            }
+        );
         self.compile_statements(body)?;
-        self.emit(Instruction::PopBlock);
-        self.emit(Instruction::Jump { target: else_block });
+        emit!(self, Instruction::PopBlock);
+        emit!(self, Instruction::Jump { target: else_block });
 
         // except handlers:
         self.switch_to_block(handler_block);
@@ -997,45 +1008,54 @@ impl Compiler {
             // check if this handler can handle the exception:
             if let Some(exc_type) = type_ {
                 // Duplicate exception for test:
-                self.emit(Instruction::Duplicate);
+                emit!(self, Instruction::Duplicate);
 
                 // Check exception type:
                 self.compile_expression(exc_type)?;
-                self.emit(Instruction::TestOperation {
-                    op: bytecode::TestOperator::ExceptionMatch,
-                });
+                emit!(
+                    self,
+                    Instruction::TestOperation {
+                        op: bytecode::TestOperator::ExceptionMatch,
+                    }
+                );
 
                 // We cannot handle this exception type:
-                self.emit(Instruction::JumpIfFalse {
-                    target: next_handler,
-                });
+                emit!(
+                    self,
+                    Instruction::JumpIfFalse {
+                        target: next_handler,
+                    }
+                );
 
                 // We have a match, store in name (except x as y)
                 if let Some(alias) = name {
                     self.store_name(alias)?
                 } else {
                     // Drop exception from top of stack:
-                    self.emit(Instruction::Pop);
+                    emit!(self, Instruction::Pop);
                 }
             } else {
                 // Catch all!
                 // Drop exception from top of stack:
-                self.emit(Instruction::Pop);
+                emit!(self, Instruction::Pop);
             }
 
             // Handler code:
             self.compile_statements(body)?;
-            self.emit(Instruction::PopException);
+            emit!(self, Instruction::PopException);
 
             if !finalbody.is_empty() {
-                self.emit(Instruction::PopBlock); // pop excepthandler block
-                                                  // We enter the finally block, without exception.
-                self.emit(Instruction::EnterFinally);
+                emit!(self, Instruction::PopBlock); // pop excepthandler block
+                                                    // We enter the finally block, without exception.
+                emit!(self, Instruction::EnterFinally);
             }
 
-            self.emit(Instruction::Jump {
-                target: finally_block,
-            });
+            emit!(
+                self,
+                Instruction::Jump {
+                    target: finally_block,
+                }
+            );
 
             // Emit a new label for the next handler
             self.switch_to_block(next_handler);
@@ -1043,9 +1063,12 @@ impl Compiler {
 
         // If code flows here, we have an unhandled exception,
         // raise the exception again!
-        self.emit(Instruction::Raise {
-            kind: bytecode::RaiseKind::Reraise,
-        });
+        emit!(
+            self,
+            Instruction::Raise {
+                kind: bytecode::RaiseKind::Reraise,
+            }
+        );
 
         // We successfully ran the try block:
         // else:
@@ -1053,17 +1076,17 @@ impl Compiler {
         self.compile_statements(orelse)?;
 
         if !finalbody.is_empty() {
-            self.emit(Instruction::PopBlock); // pop finally block
+            emit!(self, Instruction::PopBlock); // pop finally block
 
             // We enter the finallyhandler block, without return / exception.
-            self.emit(Instruction::EnterFinally);
+            emit!(self, Instruction::EnterFinally);
         }
 
         // finally:
         self.switch_to_block(finally_block);
         if !finalbody.is_empty() {
             self.compile_statements(finalbody)?;
-            self.emit(Instruction::EndFinally);
+            emit!(self, Instruction::EndFinally);
         }
 
         Ok(())
@@ -1122,7 +1145,7 @@ impl Compiler {
             }
             _ => {
                 self.emit_constant(ConstantData::None);
-                self.emit(Instruction::ReturnValue);
+                emit!(self, Instruction::ReturnValue);
             }
         }
 
@@ -1163,11 +1186,12 @@ impl Compiler {
 
         if num_annotations > 0 {
             funcflags |= bytecode::MakeFunctionFlags::ANNOTATIONS;
-            self.emit(Instruction::BuildMap {
-                size: num_annotations,
-                unpack: false,
-                for_call: false,
-            });
+            emit!(
+                self,
+                Instruction::BuildMap {
+                    size: num_annotations,
+                }
+            );
         }
 
         if self.build_closure(&code) {
@@ -1182,13 +1206,13 @@ impl Compiler {
         });
 
         // Turn code object into function object:
-        self.emit(Instruction::MakeFunction(funcflags));
+        emit!(self, Instruction::MakeFunction(funcflags));
 
-        self.emit(Instruction::Duplicate);
+        emit!(self, Instruction::Duplicate);
         self.load_docstring(doc_str);
-        self.emit(Instruction::Rotate2);
+        emit!(self, Instruction::Rotate2);
         let doc = self.name("__doc__");
-        self.emit(Instruction::StoreAttr { idx: doc });
+        emit!(self, Instruction::StoreAttr { idx: doc });
 
         self.apply_decorators(decorator_list);
 
@@ -1221,12 +1245,14 @@ impl Compiler {
             if let SymbolScope::Free = symbol.scope {
                 idx += parent_code.cellvar_cache.len();
             }
-            self.emit(Instruction::LoadClosure(idx as u32))
+            emit!(self, Instruction::LoadClosure(idx as u32))
         }
-        self.emit(Instruction::BuildTuple {
-            size: code.freevars.len() as u32,
-            unpack: false,
-        });
+        emit!(
+            self,
+            Instruction::BuildTuple {
+                size: code.freevars.len() as u32,
+            }
+        );
         true
     }
 
@@ -1266,7 +1292,7 @@ impl Compiler {
     ) -> CompileResult<()> {
         self.prepare_decorators(decorator_list)?;
 
-        self.emit(Instruction::LoadBuildClass);
+        emit!(self, Instruction::LoadBuildClass);
 
         let prev_ctx = self.ctx;
         self.ctx = CompileContext {
@@ -1294,20 +1320,20 @@ impl Compiler {
         let (doc_str, body) = split_doc(body);
 
         let dunder_name = self.name("__name__");
-        self.emit(Instruction::LoadGlobal(dunder_name));
+        emit!(self, Instruction::LoadGlobal(dunder_name));
         let dunder_module = self.name("__module__");
-        self.emit(Instruction::StoreLocal(dunder_module));
+        emit!(self, Instruction::StoreLocal(dunder_module));
         self.emit_constant(ConstantData::Str {
             value: qualified_name,
         });
         let qualname = self.name("__qualname__");
-        self.emit(Instruction::StoreLocal(qualname));
+        emit!(self, Instruction::StoreLocal(qualname));
         self.load_docstring(doc_str);
         let doc = self.name("__doc__");
-        self.emit(Instruction::StoreLocal(doc));
+        emit!(self, Instruction::StoreLocal(doc));
         // setup annotations
         if Self::find_ann(body) {
-            self.emit(Instruction::SetupAnnotation);
+            emit!(self, Instruction::SetupAnnotation);
         }
         self.compile_statements(body)?;
 
@@ -1320,15 +1346,15 @@ impl Compiler {
             .position(|var| *var == "__class__");
 
         if let Some(classcell_idx) = classcell_idx {
-            self.emit(Instruction::LoadClosure(classcell_idx as u32));
-            self.emit(Instruction::Duplicate);
+            emit!(self, Instruction::LoadClosure(classcell_idx as u32));
+            emit!(self, Instruction::Duplicate);
             let classcell = self.name("__classcell__");
-            self.emit(Instruction::StoreLocal(classcell));
+            emit!(self, Instruction::StoreLocal(classcell));
         } else {
             self.emit_constant(ConstantData::None);
         }
 
-        self.emit(Instruction::ReturnValue);
+        emit!(self, Instruction::ReturnValue);
 
         let code = self.pop_code_object();
 
@@ -1351,14 +1377,14 @@ impl Compiler {
         });
 
         // Turn code object into function object:
-        self.emit(Instruction::MakeFunction(funcflags));
+        emit!(self, Instruction::MakeFunction(funcflags));
 
         self.emit_constant(ConstantData::Str {
             value: name.to_owned(),
         });
 
         let call = self.compile_call_inner(2, bases, keywords)?;
-        self.emit(call.normal_call());
+        self.compile_normal_call(call);
 
         self.apply_decorators(decorator_list);
 
@@ -1386,9 +1412,7 @@ impl Compiler {
         let else_block = self.new_block();
         let after_block = self.new_block();
 
-        self.emit(Instruction::SetupLoop {
-            break_target: after_block,
-        });
+        emit!(self, Instruction::SetupLoop);
         self.switch_to_block(while_block);
 
         self.compile_jump_if(test, false, else_block)?;
@@ -1396,11 +1420,14 @@ impl Compiler {
         let was_in_loop = self.ctx.loop_data.replace((while_block, after_block));
         self.compile_statements(body)?;
         self.ctx.loop_data = was_in_loop;
-        self.emit(Instruction::Jump {
-            target: while_block,
-        });
+        emit!(
+            self,
+            Instruction::Jump {
+                target: while_block,
+            }
+        );
         self.switch_to_block(else_block);
-        self.emit(Instruction::PopBlock);
+        emit!(self, Instruction::PopBlock);
         self.compile_statements(orelse)?;
         self.switch_to_block(after_block);
         Ok(())
@@ -1424,13 +1451,13 @@ impl Compiler {
 
             self.set_source_location(with_location);
             if is_async {
-                self.emit(Instruction::BeforeAsyncWith);
-                self.emit(Instruction::GetAwaitable);
+                emit!(self, Instruction::BeforeAsyncWith);
+                emit!(self, Instruction::GetAwaitable);
                 self.emit_constant(ConstantData::None);
-                self.emit(Instruction::YieldFrom);
-                self.emit(Instruction::SetupAsyncWith { end: final_block });
+                emit!(self, Instruction::YieldFrom);
+                emit!(self, Instruction::SetupAsyncWith { end: final_block });
             } else {
-                self.emit(Instruction::SetupWith { end: final_block });
+                emit!(self, Instruction::SetupWith { end: final_block });
             }
 
             match &item.optional_vars {
@@ -1439,7 +1466,7 @@ impl Compiler {
                     self.compile_store(var)?;
                 }
                 None => {
-                    self.emit(Instruction::Pop);
+                    emit!(self, Instruction::Pop);
                 }
             }
             final_block
@@ -1458,20 +1485,20 @@ impl Compiler {
         // sort of "stack up" the layers of with blocks:
         // with a, b: body -> start_with(a) start_with(b) body() end_with(b) end_with(a)
         self.set_source_location(with_location);
-        self.emit(Instruction::PopBlock);
+        emit!(self, Instruction::PopBlock);
 
-        self.emit(Instruction::EnterFinally);
+        emit!(self, Instruction::EnterFinally);
 
         self.switch_to_block(final_block);
-        self.emit(Instruction::WithCleanupStart);
+        emit!(self, Instruction::WithCleanupStart);
 
         if is_async {
-            self.emit(Instruction::GetAwaitable);
+            emit!(self, Instruction::GetAwaitable);
             self.emit_constant(ConstantData::None);
-            self.emit(Instruction::YieldFrom);
+            emit!(self, Instruction::YieldFrom);
         }
 
-        self.emit(Instruction::WithCleanupFinish);
+        emit!(self, Instruction::WithCleanupFinish);
 
         Ok(())
     }
@@ -1489,31 +1516,32 @@ impl Compiler {
         let else_block = self.new_block();
         let after_block = self.new_block();
 
-        self.emit(Instruction::SetupLoop {
-            break_target: after_block,
-        });
+        emit!(self, Instruction::SetupLoop);
 
         // The thing iterated:
         self.compile_expression(iter)?;
 
         if is_async {
-            self.emit(Instruction::GetAIter);
+            emit!(self, Instruction::GetAIter);
 
             self.switch_to_block(for_block);
-            self.emit(Instruction::SetupExcept {
-                handler: else_block,
-            });
-            self.emit(Instruction::GetANext);
+            emit!(
+                self,
+                Instruction::SetupExcept {
+                    handler: else_block,
+                }
+            );
+            emit!(self, Instruction::GetANext);
             self.emit_constant(ConstantData::None);
-            self.emit(Instruction::YieldFrom);
+            emit!(self, Instruction::YieldFrom);
             self.compile_store(target)?;
-            self.emit(Instruction::PopBlock);
+            emit!(self, Instruction::PopBlock);
         } else {
             // Retrieve Iterator
-            self.emit(Instruction::GetIter);
+            emit!(self, Instruction::GetIter);
 
             self.switch_to_block(for_block);
-            self.emit(Instruction::ForIter { target: else_block });
+            emit!(self, Instruction::ForIter { target: else_block });
 
             // Start of loop iteration, set targets:
             self.compile_store(target)?;
@@ -1522,13 +1550,13 @@ impl Compiler {
         let was_in_loop = self.ctx.loop_data.replace((for_block, after_block));
         self.compile_statements(body)?;
         self.ctx.loop_data = was_in_loop;
-        self.emit(Instruction::Jump { target: for_block });
+        emit!(self, Instruction::Jump { target: for_block });
 
         self.switch_to_block(else_block);
         if is_async {
-            self.emit(Instruction::EndAsyncFor);
+            emit!(self, Instruction::EndAsyncFor);
         }
-        self.emit(Instruction::PopBlock);
+        emit!(self, Instruction::PopBlock);
         self.compile_statements(orelse)?;
 
         self.switch_to_block(after_block);
@@ -1557,37 +1585,19 @@ impl Compiler {
         let (last_op, mid_ops) = ops.split_last().unwrap();
         let (last_val, mid_vals) = vals.split_last().unwrap();
 
-        let compile_cmpop = |op: &ast::Cmpop| match op {
-            ast::Cmpop::Eq => Instruction::CompareOperation {
-                op: bytecode::ComparisonOperator::Equal,
-            },
-            ast::Cmpop::NotEq => Instruction::CompareOperation {
-                op: bytecode::ComparisonOperator::NotEqual,
-            },
-            ast::Cmpop::Lt => Instruction::CompareOperation {
-                op: bytecode::ComparisonOperator::Less,
-            },
-            ast::Cmpop::LtE => Instruction::CompareOperation {
-                op: bytecode::ComparisonOperator::LessOrEqual,
-            },
-            ast::Cmpop::Gt => Instruction::CompareOperation {
-                op: bytecode::ComparisonOperator::Greater,
-            },
-            ast::Cmpop::GtE => Instruction::CompareOperation {
-                op: bytecode::ComparisonOperator::GreaterOrEqual,
-            },
-            ast::Cmpop::In => Instruction::TestOperation {
-                op: bytecode::TestOperator::In,
-            },
-            ast::Cmpop::NotIn => Instruction::TestOperation {
-                op: bytecode::TestOperator::NotIn,
-            },
-            ast::Cmpop::Is => Instruction::TestOperation {
-                op: bytecode::TestOperator::Is,
-            },
-            ast::Cmpop::IsNot => Instruction::TestOperation {
-                op: bytecode::TestOperator::IsNot,
-            },
+        use bytecode::ComparisonOperator::*;
+        use bytecode::TestOperator::*;
+        let compile_cmpop = |c: &mut Self, op: &ast::Cmpop| match op {
+            ast::Cmpop::Eq => emit!(c, Instruction::CompareOperation { op: Equal }),
+            ast::Cmpop::NotEq => emit!(c, Instruction::CompareOperation { op: NotEqual }),
+            ast::Cmpop::Lt => emit!(c, Instruction::CompareOperation { op: Less }),
+            ast::Cmpop::LtE => emit!(c, Instruction::CompareOperation { op: LessOrEqual }),
+            ast::Cmpop::Gt => emit!(c, Instruction::CompareOperation { op: Greater }),
+            ast::Cmpop::GtE => emit!(c, Instruction::CompareOperation { op: GreaterOrEqual }),
+            ast::Cmpop::In => emit!(c, Instruction::TestOperation { op: In }),
+            ast::Cmpop::NotIn => emit!(c, Instruction::TestOperation { op: NotIn }),
+            ast::Cmpop::Is => emit!(c, Instruction::TestOperation { op: Is }),
+            ast::Cmpop::IsNot => emit!(c, Instruction::TestOperation { op: IsNot }),
         };
 
         // a == b == c == d
@@ -1613,32 +1623,38 @@ impl Compiler {
         for (op, val) in mid_ops.iter().zip(mid_vals) {
             self.compile_expression(val)?;
             // store rhs for the next comparison in chain
-            self.emit(Instruction::Duplicate);
-            self.emit(Instruction::Rotate3);
+            emit!(self, Instruction::Duplicate);
+            emit!(self, Instruction::Rotate3);
 
-            self.emit(compile_cmpop(op));
+            compile_cmpop(self, op);
 
             // if comparison result is false, we break with this value; if true, try the next one.
             if let Some((break_block, _)) = end_blocks {
-                self.emit(Instruction::JumpIfFalseOrPop {
-                    target: break_block,
-                });
+                emit!(
+                    self,
+                    Instruction::JumpIfFalseOrPop {
+                        target: break_block,
+                    }
+                );
             }
         }
 
         // handle the last comparison
         self.compile_expression(last_val)?;
-        self.emit(compile_cmpop(last_op));
+        compile_cmpop(self, last_op);
 
         if let Some((break_block, after_block)) = end_blocks {
-            self.emit(Instruction::Jump {
-                target: after_block,
-            });
+            emit!(
+                self,
+                Instruction::Jump {
+                    target: after_block,
+                }
+            );
 
             // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs.
             self.switch_to_block(break_block);
-            self.emit(Instruction::Rotate2);
-            self.emit(Instruction::Pop);
+            emit!(self, Instruction::Rotate2);
+            emit!(self, Instruction::Pop);
 
             self.switch_to_block(after_block);
         }
@@ -1679,14 +1695,14 @@ impl Compiler {
         if let ast::ExprKind::Name { id, .. } = &target.node {
             // Store as dict entry in __annotations__ dict:
             let annotations = self.name("__annotations__");
-            self.emit(Instruction::LoadNameAny(annotations));
+            emit!(self, Instruction::LoadNameAny(annotations));
             self.emit_constant(ConstantData::Str {
                 value: self.mangle(id).into_owned(),
             });
-            self.emit(Instruction::StoreSubscript);
+            emit!(self, Instruction::StoreSubscript);
         } else {
             // Drop annotation if not assigned to simple identifier.
-            self.emit(Instruction::Pop);
+            emit!(self, Instruction::Pop);
         }
 
         Ok(())
@@ -1698,13 +1714,13 @@ impl Compiler {
             ast::ExprKind::Subscript { value, slice, .. } => {
                 self.compile_expression(value)?;
                 self.compile_expression(slice)?;
-                self.emit(Instruction::StoreSubscript);
+                emit!(self, Instruction::StoreSubscript);
             }
             ast::ExprKind::Attribute { value, attr, .. } => {
                 self.check_forbidden_name(attr, NameUsage::Store)?;
                 self.compile_expression(value)?;
                 let idx = self.name(attr);
-                self.emit(Instruction::StoreAttr { idx });
+                emit!(self, Instruction::StoreAttr { idx });
             }
             ast::ExprKind::List { elts, .. } | ast::ExprKind::Tuple { elts, .. } => {
                 let mut seen_star = false;
@@ -1725,15 +1741,19 @@ impl Compiler {
                                         target.location,
                                     )
                                 })?;
-                            self.emit(Instruction::UnpackEx { before, after });
+                            let args = bytecode::UnpackExArgs { before, after };
+                            emit!(self, Instruction::UnpackEx { args });
                         }
                     }
                 }
 
                 if !seen_star {
-                    self.emit(Instruction::UnpackSequence {
-                        size: elts.len() as u32,
-                    });
+                    emit!(
+                        self,
+                        Instruction::UnpackSequence {
+                            size: elts.len() as u32,
+                        }
+                    );
                 }
 
                 for element in elts {
@@ -1777,16 +1797,16 @@ impl Compiler {
             ast::ExprKind::Subscript { value, slice, .. } => {
                 self.compile_expression(value)?;
                 self.compile_expression(slice)?;
-                self.emit(Instruction::Duplicate2);
-                self.emit(Instruction::Subscript);
+                emit!(self, Instruction::Duplicate2);
+                emit!(self, Instruction::Subscript);
                 AugAssignKind::Subscript
             }
             ast::ExprKind::Attribute { value, attr, .. } => {
                 self.check_forbidden_name(attr, NameUsage::Store)?;
                 self.compile_expression(value)?;
-                self.emit(Instruction::Duplicate);
+                emit!(self, Instruction::Duplicate);
                 let idx = self.name(attr);
-                self.emit(Instruction::LoadAttr { idx });
+                emit!(self, Instruction::LoadAttr { idx });
                 AugAssignKind::Attr { idx }
             }
             _ => {
@@ -1804,13 +1824,13 @@ impl Compiler {
             }
             AugAssignKind::Subscript => {
                 // stack: CONTAINER SLICE RESULT
-                self.emit(Instruction::Rotate3);
-                self.emit(Instruction::StoreSubscript);
+                emit!(self, Instruction::Rotate3);
+                emit!(self, Instruction::StoreSubscript);
             }
             AugAssignKind::Attr { idx } => {
                 // stack: CONTAINER RESULT
-                self.emit(Instruction::Rotate2);
-                self.emit(Instruction::StoreAttr { idx });
+                emit!(self, Instruction::Rotate2);
+                emit!(self, Instruction::StoreAttr { idx });
             }
         }
 
@@ -1833,12 +1853,11 @@ impl Compiler {
             ast::Operator::BitXor => bytecode::BinaryOperator::Xor,
             ast::Operator::BitAnd => bytecode::BinaryOperator::And,
         };
-        let ins = if inplace {
-            Instruction::BinaryOperationInplace { op }
+        if inplace {
+            emit!(self, Instruction::BinaryOperationInplace { op })
         } else {
-            Instruction::BinaryOperation { op }
-        };
-        self.emit(ins);
+            emit!(self, Instruction::BinaryOperation { op })
+        }
     }
 
     /// Implement boolean short circuit evaluation logic.
@@ -1913,13 +1932,19 @@ impl Compiler {
                 // Fall back case which always will work!
                 self.compile_expression(expression)?;
                 if condition {
-                    self.emit(Instruction::JumpIfTrue {
-                        target: target_block,
-                    });
+                    emit!(
+                        self,
+                        Instruction::JumpIfTrue {
+                            target: target_block,
+                        }
+                    );
                 } else {
-                    self.emit(Instruction::JumpIfFalse {
-                        target: target_block,
-                    });
+                    emit!(
+                        self,
+                        Instruction::JumpIfFalse {
+                            target: target_block,
+                        }
+                    );
                 }
             }
         }
@@ -1937,14 +1962,20 @@ impl Compiler {
 
             match op {
                 ast::Boolop::And => {
-                    self.emit(Instruction::JumpIfFalseOrPop {
-                        target: after_block,
-                    });
+                    emit!(
+                        self,
+                        Instruction::JumpIfFalseOrPop {
+                            target: after_block,
+                        }
+                    );
                 }
                 ast::Boolop::Or => {
-                    self.emit(Instruction::JumpIfTrueOrPop {
-                        target: after_block,
-                    });
+                    emit!(
+                        self,
+                        Instruction::JumpIfTrueOrPop {
+                            target: after_block,
+                        }
+                    );
                 }
             }
         }
@@ -1964,15 +1995,11 @@ impl Compiler {
             self.compile_expression(value)?;
             size += 1;
         }
-        self.emit(Instruction::BuildMap {
-            size,
-            unpack: false,
-            for_call: false,
-        });
+        emit!(self, Instruction::BuildMap { size });
 
         for value in unpacked_values {
             self.compile_expression(value)?;
-            self.emit(Instruction::DictUpdate);
+            emit!(self, Instruction::DictUpdate);
         }
 
         Ok(())
@@ -2000,25 +2027,24 @@ impl Compiler {
             Subscript { value, slice, .. } => {
                 self.compile_expression(value)?;
                 self.compile_expression(slice)?;
-                self.emit(Instruction::Subscript);
+                emit!(self, Instruction::Subscript);
             }
             UnaryOp { op, operand } => {
                 self.compile_expression(operand)?;
 
                 // Perform operation:
-                let i = match op {
+                let op = match op {
                     ast::Unaryop::UAdd => bytecode::UnaryOperator::Plus,
                     ast::Unaryop::USub => bytecode::UnaryOperator::Minus,
                     ast::Unaryop::Not => bytecode::UnaryOperator::Not,
                     ast::Unaryop::Invert => bytecode::UnaryOperator::Invert,
                 };
-                let i = Instruction::UnaryOperation { op: i };
-                self.emit(i);
+                emit!(self, Instruction::UnaryOperation { op });
             }
             Attribute { value, attr, .. } => {
                 self.compile_expression(value)?;
                 let idx = self.name(attr);
-                self.emit(Instruction::LoadAttr { idx });
+                emit!(self, Instruction::LoadAttr { idx });
             }
             Compare {
                 left,
@@ -2032,15 +2058,27 @@ impl Compiler {
             }
             List { elts, .. } => {
                 let (size, unpack) = self.gather_elements(0, elts)?;
-                self.emit(Instruction::BuildList { size, unpack });
+                if unpack {
+                    emit!(self, Instruction::BuildListUnpack { size });
+                } else {
+                    emit!(self, Instruction::BuildList { size });
+                }
             }
             Tuple { elts, .. } => {
                 let (size, unpack) = self.gather_elements(0, elts)?;
-                self.emit(Instruction::BuildTuple { size, unpack });
+                if unpack {
+                    emit!(self, Instruction::BuildTupleUnpack { size });
+                } else {
+                    emit!(self, Instruction::BuildTuple { size });
+                }
             }
             Set { elts, .. } => {
                 let (size, unpack) = self.gather_elements(0, elts)?;
-                self.emit(Instruction::BuildSet { size, unpack });
+                if unpack {
+                    emit!(self, Instruction::BuildSetUnpack { size });
+                } else {
+                    emit!(self, Instruction::BuildSet { size });
+                }
             }
             Dict { keys, values } => {
                 self.compile_dict(keys, values)?;
@@ -2059,7 +2097,7 @@ impl Compiler {
                     self.compile_expression(step)?;
                 }
                 let step = step.is_some();
-                self.emit(Instruction::BuildSlice { step });
+                emit!(self, Instruction::BuildSlice { step });
             }
             Yield { value } => {
                 if !self.ctx.in_func() {
@@ -2070,16 +2108,16 @@ impl Compiler {
                     Some(expression) => self.compile_expression(expression)?,
                     Option::None => self.emit_constant(ConstantData::None),
                 };
-                self.emit(Instruction::YieldValue);
+                emit!(self, Instruction::YieldValue);
             }
             Await { value } => {
                 if self.ctx.func != FunctionContext::AsyncFunction {
                     return Err(self.error(CodegenErrorType::InvalidAwait));
                 }
                 self.compile_expression(value)?;
-                self.emit(Instruction::GetAwaitable);
+                emit!(self, Instruction::GetAwaitable);
                 self.emit_constant(ConstantData::None);
-                self.emit(Instruction::YieldFrom);
+                emit!(self, Instruction::YieldFrom);
             }
             YieldFrom { value } => {
                 match self.ctx.func {
@@ -2093,9 +2131,9 @@ impl Compiler {
                 }
                 self.mark_generator();
                 self.compile_expression(value)?;
-                self.emit(Instruction::GetIter);
+                emit!(self, Instruction::GetIter);
                 self.emit_constant(ConstantData::None);
-                self.emit(Instruction::YieldFrom);
+                emit!(self, Instruction::YieldFrom);
             }
             ast::ExprKind::JoinedStr { values } => {
                 if let Some(value) = try_get_constant_string(values) {
@@ -2104,9 +2142,12 @@ impl Compiler {
                     for value in values {
                         self.compile_expression(value)?;
                     }
-                    self.emit(Instruction::BuildString {
-                        size: values.len() as u32,
-                    })
+                    emit!(
+                        self,
+                        Instruction::BuildString {
+                            size: values.len() as u32,
+                        }
+                    )
                 }
             }
             ast::ExprKind::FormattedValue {
@@ -2121,9 +2162,13 @@ impl Compiler {
                     }),
                 };
                 self.compile_expression(value)?;
-                self.emit(Instruction::FormatValue {
-                    conversion: (*conversion).try_into().expect("invalid conversion flag"),
-                });
+                emit!(
+                    self,
+                    Instruction::FormatValue {
+                        conversion: bytecode::ConversionFlag::try_from(*conversion)
+                            .expect("invalid conversion flag"),
+                    },
+                );
             }
             Name { id, .. } => self.load_name(id)?,
             Lambda { args, body } => {
@@ -2143,7 +2188,7 @@ impl Compiler {
                     .insert_full(ConstantData::None);
 
                 self.compile_expression(body)?;
-                self.emit(Instruction::ReturnValue);
+                emit!(self, Instruction::ReturnValue);
                 let code = self.pop_code_object();
                 if self.build_closure(&code) {
                     funcflags |= bytecode::MakeFunctionFlags::CLOSURE;
@@ -2153,7 +2198,7 @@ impl Compiler {
                 });
                 self.emit_constant(ConstantData::Str { value: name });
                 // Turn code object into function object:
-                self.emit(Instruction::MakeFunction(funcflags));
+                emit!(self, Instruction::MakeFunction(funcflags));
 
                 self.ctx = prev_ctx;
             }
@@ -2161,15 +2206,17 @@ impl Compiler {
                 self.compile_comprehension(
                     "<listcomp>",
                     Some(Instruction::BuildList {
-                        size: 0,
-                        unpack: false,
+                        size: OpArgMarker::marker(),
                     }),
                     generators,
                     &|compiler| {
                         compiler.compile_comprehension_element(elt)?;
-                        compiler.emit(Instruction::ListAppend {
-                            i: generators.len() as u32,
-                        });
+                        emit!(
+                            compiler,
+                            Instruction::ListAppend {
+                                i: generators.len() as u32,
+                            }
+                        );
                         Ok(())
                     },
                 )?;
@@ -2178,15 +2225,17 @@ impl Compiler {
                 self.compile_comprehension(
                     "<setcomp>",
                     Some(Instruction::BuildSet {
-                        size: 0,
-                        unpack: false,
+                        size: OpArgMarker::marker(),
                     }),
                     generators,
                     &|compiler| {
                         compiler.compile_comprehension_element(elt)?;
-                        compiler.emit(Instruction::SetAdd {
-                            i: generators.len() as u32,
-                        });
+                        emit!(
+                            compiler,
+                            Instruction::SetAdd {
+                                i: generators.len() as u32,
+                            }
+                        );
                         Ok(())
                     },
                 )?;
@@ -2199,9 +2248,7 @@ impl Compiler {
                 self.compile_comprehension(
                     "<dictcomp>",
                     Some(Instruction::BuildMap {
-                        size: 0,
-                        for_call: false,
-                        unpack: false,
+                        size: OpArgMarker::marker(),
                     }),
                     generators,
                     &|compiler| {
@@ -2209,9 +2256,12 @@ impl Compiler {
                         compiler.compile_expression(key)?;
                         compiler.compile_expression(value)?;
 
-                        compiler.emit(Instruction::MapAdd {
-                            i: generators.len() as u32,
-                        });
+                        emit!(
+                            compiler,
+                            Instruction::MapAdd {
+                                i: generators.len() as u32,
+                            }
+                        );
 
                         Ok(())
                     },
@@ -2221,8 +2271,8 @@ impl Compiler {
                 self.compile_comprehension("<genexpr>", None, generators, &|compiler| {
                     compiler.compile_comprehension_element(elt)?;
                     compiler.mark_generator();
-                    compiler.emit(Instruction::YieldValue);
-                    compiler.emit(Instruction::Pop);
+                    emit!(compiler, Instruction::YieldValue);
+                    emit!(compiler, Instruction::Pop);
 
                     Ok(())
                 })?;
@@ -2237,9 +2287,12 @@ impl Compiler {
 
                 // True case
                 self.compile_expression(body)?;
-                self.emit(Instruction::Jump {
-                    target: after_block,
-                });
+                emit!(
+                    self,
+                    Instruction::Jump {
+                        target: after_block,
+                    }
+                );
 
                 // False case
                 self.switch_to_block(else_block);
@@ -2251,7 +2304,7 @@ impl Compiler {
 
             NamedExpr { target, value } => {
                 self.compile_expression(value)?;
-                self.emit(Instruction::Duplicate);
+                emit!(self, Instruction::Duplicate);
                 self.compile_store(target)?;
             }
         }
@@ -2278,20 +2331,12 @@ impl Compiler {
                         subsize += 1;
                     }
                 }
-                self.emit(Instruction::BuildMap {
-                    size: subsize,
-                    unpack: false,
-                    for_call: false,
-                });
+                emit!(self, Instruction::BuildMap { size: subsize });
                 size += 1;
             }
         }
         if size > 1 {
-            self.emit(Instruction::BuildMap {
-                size,
-                unpack: true,
-                for_call: true,
-            });
+            emit!(self, Instruction::BuildMapForCall { size });
         }
         Ok(())
     }
@@ -2305,21 +2350,40 @@ impl Compiler {
         let method = if let ast::ExprKind::Attribute { value, attr, .. } = &func.node {
             self.compile_expression(value)?;
             let idx = self.name(attr);
-            self.emit(Instruction::LoadMethod { idx });
+            emit!(self, Instruction::LoadMethod { idx });
             true
         } else {
             self.compile_expression(func)?;
             false
         };
         let call = self.compile_call_inner(0, args, keywords)?;
-        self.emit(if method {
-            call.method_call()
+        if method {
+            self.compile_method_call(call)
         } else {
-            call.normal_call()
-        });
+            self.compile_normal_call(call)
+        }
         Ok(())
     }
 
+    fn compile_normal_call(&mut self, ty: CallType) {
+        match ty {
+            CallType::Positional { nargs } => {
+                emit!(self, Instruction::CallFunctionPositional { nargs })
+            }
+            CallType::Keyword { nargs } => emit!(self, Instruction::CallFunctionKeyword { nargs }),
+            CallType::Ex { has_kwargs } => emit!(self, Instruction::CallFunctionEx { has_kwargs }),
+        }
+    }
+    fn compile_method_call(&mut self, ty: CallType) {
+        match ty {
+            CallType::Positional { nargs } => {
+                emit!(self, Instruction::CallMethodPositional { nargs })
+            }
+            CallType::Keyword { nargs } => emit!(self, Instruction::CallMethodKeyword { nargs }),
+            CallType::Ex { has_kwargs } => emit!(self, Instruction::CallMethodEx { has_kwargs }),
+        }
+    }
+
     fn compile_call_inner(
         &mut self,
         additional_positional: u32,
@@ -2340,7 +2404,11 @@ impl Compiler {
 
         let call = if unpack || has_double_star {
             // Create a tuple with positional args:
-            self.emit(Instruction::BuildTuple { size, unpack });
+            if unpack {
+                emit!(self, Instruction::BuildTupleUnpack { size });
+            } else {
+                emit!(self, Instruction::BuildTuple { size });
+            }
 
             // Create an optional map with kw-args:
             let has_kwargs = !keywords.is_empty();
@@ -2389,10 +2457,7 @@ impl Compiler {
             let mut size = 0;
 
             if before > 0 {
-                self.emit(Instruction::BuildTuple {
-                    size: before,
-                    unpack: false,
-                });
+                emit!(self, Instruction::BuildTuple { size: before });
                 size += 1;
             }
 
@@ -2416,10 +2481,7 @@ impl Compiler {
                 if starred {
                     size += run_size
                 } else {
-                    self.emit(Instruction::BuildTuple {
-                        size: run_size,
-                        unpack: false,
-                    });
+                    emit!(self, Instruction::BuildTuple { size: run_size });
                     size += 1
                 }
             }
@@ -2478,7 +2540,7 @@ impl Compiler {
         let return_none = init_collection.is_none();
         // Create empty object of proper type:
         if let Some(init_collection) = init_collection {
-            self.emit(init_collection)
+            self._emit(init_collection, OpArg(0), ir::BlockIdx::NULL)
         }
 
         let mut loop_labels = vec![];
@@ -2492,21 +2554,24 @@ impl Compiler {
 
             if loop_labels.is_empty() {
                 // Load iterator onto stack (passed as first argument):
-                self.emit(Instruction::LoadFast(arg0));
+                emit!(self, Instruction::LoadFast(arg0));
             } else {
                 // Evaluate iterated item:
                 self.compile_expression(&generator.iter)?;
 
                 // Get iterator / turn item into an iterator
-                self.emit(Instruction::GetIter);
+                emit!(self, Instruction::GetIter);
             }
 
             loop_labels.push((loop_block, after_block));
 
             self.switch_to_block(loop_block);
-            self.emit(Instruction::ForIter {
-                target: after_block,
-            });
+            emit!(
+                self,
+                Instruction::ForIter {
+                    target: after_block,
+                }
+            );
 
             self.compile_store(&generator.target)?;
 
@@ -2520,7 +2585,7 @@ impl Compiler {
 
         for (loop_block, after_block) in loop_labels.iter().rev().copied() {
             // Repeat:
-            self.emit(Instruction::Jump { target: loop_block });
+            emit!(self, Instruction::Jump { target: loop_block });
 
             // End of for loop:
             self.switch_to_block(after_block);
@@ -2531,7 +2596,7 @@ impl Compiler {
         }
 
         // Return freshly filled list:
-        self.emit(Instruction::ReturnValue);
+        emit!(self, Instruction::ReturnValue);
 
         // Fetch code for listcomp function:
         let code = self.pop_code_object();
@@ -2554,16 +2619,16 @@ impl Compiler {
         });
 
         // Turn code object into function object:
-        self.emit(Instruction::MakeFunction(funcflags));
+        emit!(self, Instruction::MakeFunction(funcflags));
 
         // Evaluate iterated item:
         self.compile_expression(&generators[0].iter)?;
 
         // Get iterator / turn item into an iterator
-        self.emit(Instruction::GetIter);
+        emit!(self, Instruction::GetIter);
 
         // Call just created <listcomp> function:
-        self.emit(Instruction::CallFunctionPositional { nargs: 1 });
+        emit!(self, Instruction::CallFunctionPositional { nargs: 1 });
         Ok(())
     }
 
@@ -2587,12 +2652,28 @@ impl Compiler {
     }
 
     // Low level helper functions:
-    fn emit(&mut self, instr: Instruction) {
+    fn _emit(&mut self, instr: Instruction, arg: OpArg, target: ir::BlockIdx) {
         let location = compile_location(&self.current_source_location);
         // TODO: insert source filename
-        self.current_block()
-            .instructions
-            .push(ir::InstructionInfo { instr, location });
+        self.current_block().instructions.push(ir::InstructionInfo {
+            instr,
+            arg,
+            target,
+            location,
+        });
+    }
+
+    fn emit_noarg(&mut self, ins: Instruction) {
+        self._emit(ins, OpArg::null(), ir::BlockIdx::NULL)
+    }
+
+    fn emit_arg<A: OpArgType, T: EmitArg<A>>(
+        &mut self,
+        arg: T,
+        f: impl FnOnce(OpArgMarker<A>) -> Instruction,
+    ) {
+        let (op, arg, target) = arg.emit(f);
+        self._emit(op, arg, target)
     }
 
     // fn block_done()
@@ -2600,7 +2681,7 @@ impl Compiler {
     fn emit_constant(&mut self, constant: ConstantData) {
         let info = self.current_codeinfo();
         let idx = info.constants.insert_full(constant).0 as u32;
-        self.emit(Instruction::LoadConst { idx })
+        self.emit_arg(idx, |idx| Instruction::LoadConst { idx })
     }
 
     fn current_codeinfo(&mut self) -> &mut ir::CodeInfo {
@@ -2609,12 +2690,12 @@ impl Compiler {
 
     fn current_block(&mut self) -> &mut ir::Block {
         let info = self.current_codeinfo();
-        &mut info.blocks[info.current_block.0 as usize]
+        &mut info.blocks[info.current_block]
     }
 
     fn new_block(&mut self) -> ir::BlockIdx {
         let code = self.current_codeinfo();
-        let idx = bytecode::Label(code.blocks.len() as u32);
+        let idx = ir::BlockIdx(code.blocks.len() as u32);
         code.blocks.push(ir::Block::default());
         idx
     }
@@ -2623,8 +2704,8 @@ impl Compiler {
         let code = self.current_codeinfo();
         let prev = code.current_block;
         assert_eq!(
-            code.blocks[block.0 as usize].next.0,
-            u32::MAX,
+            code.blocks[block].next,
+            ir::BlockIdx::NULL,
             "switching to completed block"
         );
         let prev_block = &mut code.blocks[prev.0 as usize];
@@ -2654,6 +2735,30 @@ impl Compiler {
     }
 }
 
+trait EmitArg<Arg: OpArgType> {
+    fn emit(
+        self,
+        f: impl FnOnce(OpArgMarker<Arg>) -> Instruction,
+    ) -> (Instruction, OpArg, ir::BlockIdx);
+}
+impl<T: OpArgType> EmitArg<T> for T {
+    fn emit(
+        self,
+        f: impl FnOnce(OpArgMarker<T>) -> Instruction,
+    ) -> (Instruction, OpArg, ir::BlockIdx) {
+        let (marker, arg) = OpArgMarker::new(self);
+        (f(marker), arg, ir::BlockIdx::NULL)
+    }
+}
+impl EmitArg<bytecode::Label> for ir::BlockIdx {
+    fn emit(
+        self,
+        f: impl FnOnce(OpArgMarker<bytecode::Label>) -> Instruction,
+    ) -> (Instruction, OpArg, ir::BlockIdx) {
+        (f(OpArgMarker::marker()), OpArg::null(), self)
+    }
+}
+
 fn split_doc(body: &[ast::Stmt]) -> (Option<String>, &[ast::Stmt]) {
     if let Some((val, body_rest)) = body.split_first() {
         if let ast::StmtKind::Expr { value } = &val.node {
diff --git a/codegen/src/ir.rs b/codegen/src/ir.rs
index c1ed1dd7..43d7be0f 100644
--- a/codegen/src/ir.rs
+++ b/codegen/src/ir.rs
@@ -1,12 +1,47 @@
+use std::ops;
+
 use crate::IndexSet;
-use rustpython_compiler_core::{CodeFlags, CodeObject, ConstantData, Instruction, Label, Location};
+use rustpython_compiler_core::{
+    CodeFlags, CodeObject, CodeUnit, ConstantData, InstrDisplayContext, Instruction, Label,
+    Location, OpArg,
+};
 
-pub type BlockIdx = Label;
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub struct BlockIdx(pub u32);
+impl BlockIdx {
+    pub const NULL: BlockIdx = BlockIdx(u32::MAX);
+    const fn idx(self) -> usize {
+        self.0 as usize
+    }
+}
+impl ops::Index<BlockIdx> for [Block] {
+    type Output = Block;
+    fn index(&self, idx: BlockIdx) -> &Block {
+        &self[idx.idx()]
+    }
+}
+impl ops::IndexMut<BlockIdx> for [Block] {
+    fn index_mut(&mut self, idx: BlockIdx) -> &mut Block {
+        &mut self[idx.idx()]
+    }
+}
+impl ops::Index<BlockIdx> for Vec<Block> {
+    type Output = Block;
+    fn index(&self, idx: BlockIdx) -> &Block {
+        &self[idx.idx()]
+    }
+}
+impl ops::IndexMut<BlockIdx> for Vec<Block> {
+    fn index_mut(&mut self, idx: BlockIdx) -> &mut Block {
+        &mut self[idx.idx()]
+    }
+}
 
-#[derive(Debug)]
+#[derive(Debug, Copy, Clone)]
 pub struct InstructionInfo {
-    /// If the instruction has a Label argument, it's actually a BlockIdx, not a code offset
     pub instr: Instruction,
+    pub arg: OpArg,
+    pub target: BlockIdx,
     pub location: Location,
 }
 
@@ -21,7 +56,7 @@ impl Default for Block {
     fn default() -> Self {
         Block {
             instructions: Vec::new(),
-            next: Label(u32::MAX),
+            next: BlockIdx::NULL,
         }
     }
 }
@@ -45,13 +80,13 @@ pub struct CodeInfo {
 }
 impl CodeInfo {
     pub fn finalize_code(mut self, optimize: u8) -> CodeObject {
-        let max_stackdepth = self.max_stackdepth();
-        let cell2arg = self.cell2arg();
-
         if optimize > 0 {
             self.dce();
         }
 
+        let max_stackdepth = self.max_stackdepth();
+        let cell2arg = self.cell2arg();
+
         let CodeInfo {
             flags,
             posonlyarg_count,
@@ -61,7 +96,7 @@ impl CodeInfo {
             first_line_number,
             obj_name,
 
-            blocks,
+            mut blocks,
             current_block: _,
             constants,
             name_cache,
@@ -70,26 +105,50 @@ impl CodeInfo {
             freevar_cache,
         } = self;
 
-        let mut num_instructions = 0;
-        let mut block_to_offset = vec![Label(0); blocks.len()];
+        let mut instructions = Vec::new();
+        let mut locations = Vec::new();
 
-        for (idx, block) in iter_blocks(&blocks) {
-            block_to_offset[idx.0 as usize] = Label(num_instructions as u32);
-            num_instructions += block.instructions.len();
-        }
+        let mut block_to_offset = vec![Label(0); blocks.len()];
+        loop {
+            let mut num_instructions = 0;
+            for (idx, block) in iter_blocks(&blocks) {
+                block_to_offset[idx.idx()] = Label(num_instructions as u32);
+                for instr in &block.instructions {
+                    num_instructions += instr.arg.instr_size()
+                }
+            }
 
-        let mut instructions = Vec::with_capacity(num_instructions);
-        let mut locations = Vec::with_capacity(num_instructions);
+            instructions.reserve_exact(num_instructions);
+            locations.reserve_exact(num_instructions);
 
-        for (_, block) in iter_blocks(&blocks) {
-            for info in &block.instructions {
-                let mut instr = info.instr.clone();
-                if let Some(l) = instr.label_arg_mut() {
-                    *l = block_to_offset[l.0 as usize];
+            let mut recompile_extended_arg = false;
+            let mut next_block = BlockIdx(0);
+            while next_block != BlockIdx::NULL {
+                let block = &mut blocks[next_block];
+                for info in &mut block.instructions {
+                    let (op, arg, target) = (info.instr, &mut info.arg, info.target);
+                    if target != BlockIdx::NULL {
+                        let new_arg = OpArg(block_to_offset[target.idx()].0);
+                        recompile_extended_arg |= new_arg.instr_size() != arg.instr_size();
+                        *arg = new_arg;
+                    }
+                    let (extras, lo_arg) = arg.split();
+                    locations.extend(std::iter::repeat(info.location).take(arg.instr_size()));
+                    instructions.extend(
+                        extras
+                            .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte))
+                            .chain([CodeUnit { op, arg: lo_arg }]),
+                    );
                 }
-                instructions.push(instr);
-                locations.push(info.location);
+                next_block = block.next;
             }
+
+            if !recompile_extended_arg {
+                break;
+            }
+
+            instructions.clear();
+            locations.clear()
         }
 
         CodeObject {
@@ -166,40 +225,48 @@ impl CodeInfo {
         let mut stack = Vec::with_capacity(self.blocks.len());
         let mut startdepths = vec![u32::MAX; self.blocks.len()];
         startdepths[0] = 0;
-        stack.push(Label(0));
+        stack.push(BlockIdx(0));
         const DEBUG: bool = false;
         'process_blocks: while let Some(block) = stack.pop() {
-            let mut depth = startdepths[block.0 as usize];
+            let mut depth = startdepths[block.idx()];
             if DEBUG {
                 eprintln!("===BLOCK {}===", block.0);
             }
-            let block = &self.blocks[block.0 as usize];
+            let block = &self.blocks[block];
             for i in &block.instructions {
                 let instr = &i.instr;
-                let effect = instr.stack_effect(false);
+                let effect = instr.stack_effect(i.arg, false);
                 if DEBUG {
-                    eprint!("{instr:?}: {depth} {effect:+} => ");
+                    let display_arg = if i.target == BlockIdx::NULL {
+                        i.arg
+                    } else {
+                        OpArg(i.target.0)
+                    };
+                    let instr_display = instr.display(display_arg, self);
+                    eprint!("{instr_display}: {depth} {effect:+} => ");
                 }
-                let new_depth = add_ui(depth, effect);
+                let new_depth = depth.checked_add_signed(effect).unwrap();
                 if DEBUG {
                     eprintln!("{new_depth}");
                 }
                 if new_depth > maxdepth {
                     maxdepth = new_depth
                 }
-                // we don't want to worry about Continue, it uses unwinding to jump to
-                // its targets and as such the stack size is taken care of in frame.rs by setting
+                // we don't want to worry about Break/Continue, they use unwinding to jump to
+                // their targets and as such the stack size is taken care of in frame.rs by setting
                 // it back to the level it was at when SetupLoop was run
-                let jump_label = instr
-                    .label_arg()
-                    .filter(|_| !matches!(instr, Instruction::Continue { .. }));
-                if let Some(&target_block) = jump_label {
-                    let effect = instr.stack_effect(true);
-                    let target_depth = add_ui(depth, effect);
+                if i.target != BlockIdx::NULL
+                    && !matches!(
+                        instr,
+                        Instruction::Continue { .. } | Instruction::Break { .. }
+                    )
+                {
+                    let effect = instr.stack_effect(i.arg, true);
+                    let target_depth = depth.checked_add_signed(effect).unwrap();
                     if target_depth > maxdepth {
                         maxdepth = target_depth
                     }
-                    stackdepth_push(&mut stack, &mut startdepths, target_block, target_depth);
+                    stackdepth_push(&mut stack, &mut startdepths, i.target, target_depth);
                 }
                 depth = new_depth;
                 if instr.unconditional_branch() {
@@ -215,23 +282,46 @@ impl CodeInfo {
     }
 }
 
-fn stackdepth_push(stack: &mut Vec<Label>, startdepths: &mut [u32], target: Label, depth: u32) {
-    let block_depth = &mut startdepths[target.0 as usize];
-    if *block_depth == u32::MAX || depth > *block_depth {
-        *block_depth = depth;
-        stack.push(target);
+impl InstrDisplayContext for CodeInfo {
+    type Constant = ConstantData;
+    fn get_constant(&self, i: usize) -> &ConstantData {
+        &self.constants[i]
+    }
+    fn get_name(&self, i: usize) -> &str {
+        self.name_cache[i].as_ref()
+    }
+    fn get_varname(&self, i: usize) -> &str {
+        self.varname_cache[i].as_ref()
+    }
+    fn get_cellname(&self, i: usize) -> &str {
+        self.cellvar_cache
+            .get_index(i)
+            .unwrap_or_else(|| &self.freevar_cache[i - self.cellvar_cache.len()])
+            .as_ref()
     }
 }
 
-fn add_ui(a: u32, b: i32) -> u32 {
-    if b < 0 {
-        a - b.wrapping_abs() as u32
-    } else {
-        a + b as u32
+fn stackdepth_push(
+    stack: &mut Vec<BlockIdx>,
+    startdepths: &mut [u32],
+    target: BlockIdx,
+    depth: u32,
+) {
+    let block_depth = &mut startdepths[target.idx()];
+    if *block_depth == u32::MAX || depth > *block_depth {
+        *block_depth = depth;
+        stack.push(target);
     }
 }
 
 fn iter_blocks(blocks: &[Block]) -> impl Iterator<Item = (BlockIdx, &Block)> + '_ {
-    let get_idx = move |i: BlockIdx| blocks.get(i.0 as usize).map(|b| (i, b));
-    std::iter::successors(get_idx(Label(0)), move |(_, b)| get_idx(b.next)) // if b.next is u32::MAX that's the end
+    let mut next = BlockIdx(0);
+    std::iter::from_fn(move || {
+        if next == BlockIdx::NULL {
+            return None;
+        }
+        let (idx, b) = (next, &blocks[next]);
+        next = b.next;
+        Some((idx, b))
+    })
 }
diff --git a/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap b/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap
index ebefb03d..0563b38e 100644
--- a/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap
+++ b/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap
@@ -2,23 +2,23 @@
 source: compiler/codegen/src/compile.rs
 expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n    with self.subTest(type=type(stop_exc)):\n        try:\n            async with woohoo():\n                raise stop_exc\n        except Exception as ex:\n            self.assertIs(ex, stop_exc)\n        else:\n            self.fail(f'{stop_exc} was suppressed')\n\")"
 ---
-  1           0 SetupLoop            (69)
+  1           0 SetupLoop
               1 LoadNameAny          (0, StopIteration)
               2 LoadConst            ("spam")
-              3 CallFunctionPositional (1)
+              3 CallFunctionPositional(1)
               4 LoadNameAny          (1, StopAsyncIteration)
               5 LoadConst            ("ham")
-              6 CallFunctionPositional (1)
-              7 BuildTuple           (2, false)
+              6 CallFunctionPositional(1)
+              7 BuildTuple           (2)
               8 GetIter
         >>    9 ForIter              (68)
              10 StoreLocal           (2, stop_exc)
 
   2          11 LoadNameAny          (3, self)
-             12 LoadMethod           (subTest)
+             12 LoadMethod           (4, subTest)
              13 LoadNameAny          (5, type)
              14 LoadNameAny          (2, stop_exc)
-             15 CallFunctionPositional (1)
+             15 CallFunctionPositional(1)
              16 LoadConst            (("type"))
              17 CallMethodKeyword    (1)
              18 SetupWith            (65)
@@ -27,7 +27,7 @@ expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyn
   3          20 SetupExcept          (40)
 
   4          21 LoadNameAny          (6, woohoo)
-             22 CallFunctionPositional (0)
+             22 CallFunctionPositional(0)
              23 BeforeAsyncWith
              24 GetAwaitable
              25 LoadConst            (None)
@@ -55,7 +55,7 @@ expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyn
              44 StoreLocal           (8, ex)
 
   7          45 LoadNameAny          (3, self)
-             46 LoadMethod           (assertIs)
+             46 LoadMethod           (9, assertIs)
              47 LoadNameAny          (8, ex)
              48 LoadNameAny          (2, stop_exc)
              49 CallMethodPositional (2)
@@ -65,7 +65,7 @@ expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyn
         >>   53 Raise                (Reraise)
 
   9     >>   54 LoadNameAny          (3, self)
-             55 LoadMethod           (fail)
+             55 LoadMethod           (10, fail)
              56 LoadConst            ("")
              57 LoadNameAny          (2, stop_exc)
              58 FormatValue          (None)
@@ -80,6 +80,6 @@ expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyn
              66 WithCleanupFinish
              67 Jump                 (9)
         >>   68 PopBlock
-        >>   69 LoadConst            (None)
+             69 LoadConst            (None)
              70 ReturnValue
 
diff --git a/core/Cargo.toml b/core/Cargo.toml
index a3dd82bb..f9cbce31 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -15,6 +15,6 @@ itertools = "0.10.3"
 lz4_flex = "0.9.2"
 num-bigint = { version = "0.4.3", features = ["serde"] }
 num-complex = { version = "0.4.0", features = ["serde"] }
+num_enum = "0.5.7"
 serde = { version = "1.0.136", features = ["derive"] }
-static_assertions = "1.1.0"
 thiserror = "1.0"
diff --git a/core/src/bytecode.rs b/core/src/bytecode.rs
index ea2477ae..b7c456e9 100644
--- a/core/src/bytecode.rs
+++ b/core/src/bytecode.rs
@@ -7,8 +7,10 @@ use bstr::ByteSlice;
 use itertools::Itertools;
 use num_bigint::BigInt;
 use num_complex::Complex64;
+use num_enum::{IntoPrimitive, TryFromPrimitive};
 use serde::{Deserialize, Serialize};
-use std::{collections::BTreeSet, fmt, hash};
+use std::marker::PhantomData;
+use std::{collections::BTreeSet, fmt, hash, mem};
 
 pub trait Constant: Sized {
     type Name: AsRef<str>;
@@ -62,7 +64,7 @@ impl ConstantBag for BasicBag {
 /// a codeobject. Also a module has a codeobject.
 #[derive(Clone, Serialize, Deserialize)]
 pub struct CodeObject<C: Constant = ConstantData> {
-    pub instructions: Box<[Instruction]>,
+    pub instructions: Box<[CodeUnit]>,
     pub locations: Box<[Location]>,
     pub flags: CodeFlags,
     pub posonlyarg_count: usize,
@@ -111,12 +113,188 @@ impl CodeFlags {
     ];
 }
 
-#[derive(Serialize, Debug, Deserialize, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)]
+/// an opcode argument that may be extended by a prior ExtendedArg
+#[derive(Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[repr(transparent)]
+pub struct OpArgByte(pub u8);
+impl OpArgByte {
+    pub const fn null() -> Self {
+        OpArgByte(0)
+    }
+}
+impl fmt::Debug for OpArgByte {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+/// a full 32-bit oparg, including any possible ExtendedArg extension
+#[derive(Copy, Clone, Debug)]
+#[repr(transparent)]
+pub struct OpArg(pub u32);
+impl OpArg {
+    pub const fn null() -> Self {
+        OpArg(0)
+    }
+
+    /// Returns how many CodeUnits a instruction with this oparg will be encoded as
+    #[inline]
+    pub fn instr_size(self) -> usize {
+        (self.0 > 0xff) as usize + (self.0 > 0xff_ff) as usize + (self.0 > 0xff_ff_ff) as usize + 1
+    }
+
+    /// returns the arg split into any necessary ExtendedArg components (in big-endian order) and
+    /// the arg for the real opcode itself
+    #[inline(always)]
+    pub fn split(self) -> (impl ExactSizeIterator<Item = OpArgByte>, OpArgByte) {
+        let mut it = self
+            .0
+            .to_le_bytes()
+            .map(OpArgByte)
+            .into_iter()
+            .take(self.instr_size());
+        let lo = it.next().unwrap();
+        (it.rev(), lo)
+    }
+}
+
+#[derive(Default, Copy, Clone)]
+#[repr(transparent)]
+pub struct OpArgState {
+    state: u32,
+}
+
+impl OpArgState {
+    #[inline(always)]
+    pub fn get(&mut self, ins: CodeUnit) -> (Instruction, OpArg) {
+        let arg = self.extend(ins.arg);
+        if ins.op != Instruction::ExtendedArg {
+            self.reset();
+        }
+        (ins.op, arg)
+    }
+    #[inline(always)]
+    pub fn extend(&mut self, arg: OpArgByte) -> OpArg {
+        self.state = self.state << 8 | u32::from(arg.0);
+        OpArg(self.state)
+    }
+    #[inline(always)]
+    pub fn reset(&mut self) {
+        self.state = 0
+    }
+}
+
+pub trait OpArgType: Copy {
+    fn from_oparg(x: u32) -> Option<Self>;
+    fn to_oparg(self) -> u32;
+}
+
+impl OpArgType for u32 {
+    #[inline(always)]
+    fn from_oparg(x: u32) -> Option<Self> {
+        Some(x)
+    }
+    #[inline(always)]
+    fn to_oparg(self) -> u32 {
+        self
+    }
+}
+
+impl OpArgType for bool {
+    #[inline(always)]
+    fn from_oparg(x: u32) -> Option<Self> {
+        Some(x != 0)
+    }
+    #[inline(always)]
+    fn to_oparg(self) -> u32 {
+        self as u32
+    }
+}
+
+macro_rules! enum_oparg {
+    ($t:ident) => {
+        impl OpArgType for $t {
+            #[inline(always)]
+            fn from_oparg(x: u32) -> Option<Self> {
+                $t::try_from_primitive(x as _).ok()
+            }
+            #[inline(always)]
+            fn to_oparg(self) -> u32 {
+                u8::from(self).into()
+            }
+        }
+    };
+}
+
+#[derive(Copy, Clone, Serialize, Deserialize)]
+#[serde(bound = "")]
+pub struct Arg<T: OpArgType>(PhantomData<T>);
+
+impl<T: OpArgType> Arg<T> {
+    #[inline]
+    pub fn marker() -> Self {
+        Arg(PhantomData)
+    }
+    #[inline]
+    pub fn new(arg: T) -> (Self, OpArg) {
+        (Self(PhantomData), OpArg(arg.to_oparg()))
+    }
+    #[inline]
+    pub fn new_single(arg: T) -> (Self, OpArgByte)
+    where
+        T: Into<u8>,
+    {
+        (Self(PhantomData), OpArgByte(arg.into()))
+    }
+    #[inline(always)]
+    pub fn get(self, arg: OpArg) -> T {
+        self.try_get(arg).unwrap()
+    }
+    #[inline(always)]
+    pub fn try_get(self, arg: OpArg) -> Option<T> {
+        T::from_oparg(arg.0)
+    }
+    #[inline(always)]
+    /// # Safety
+    /// T::from_oparg(self) must succeed
+    pub unsafe fn get_unchecked(self, arg: OpArg) -> T {
+        match T::from_oparg(arg.0) {
+            Some(t) => t,
+            None => std::hint::unreachable_unchecked(),
+        }
+    }
+}
+
+impl<T: OpArgType> PartialEq for Arg<T> {
+    fn eq(&self, _: &Self) -> bool {
+        true
+    }
+}
+impl<T: OpArgType> Eq for Arg<T> {}
+
+impl<T: OpArgType> fmt::Debug for Arg<T> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "Arg<{}>", std::any::type_name::<T>())
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)]
 #[repr(transparent)]
 // XXX: if you add a new instruction that stores a Label, make sure to add it in
-// Instruction::label_arg{,_mut}
+// Instruction::label_arg
 pub struct Label(pub u32);
 
+impl OpArgType for Label {
+    #[inline(always)]
+    fn from_oparg(x: u32) -> Option<Self> {
+        Some(Label(x))
+    }
+    #[inline(always)]
+    fn to_oparg(self) -> u32 {
+        self.0
+    }
+}
+
 impl fmt::Display for Label {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         self.0.fmt(f)
@@ -124,7 +302,7 @@ impl fmt::Display for Label {
 }
 
 /// Transforms a value prior to formatting it.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
 #[repr(u8)]
 pub enum ConversionFlag {
     /// No conversion
@@ -136,37 +314,36 @@ pub enum ConversionFlag {
     /// Converts by calling `repr(<value>)`.
     Repr = b'r',
 }
+enum_oparg!(ConversionFlag);
 
 impl TryFrom<usize> for ConversionFlag {
     type Error = usize;
     fn try_from(b: usize) -> Result<Self, Self::Error> {
-        let b = b.try_into().map_err(|_| b)?;
-        match b {
-            0 => Ok(Self::None),
-            b's' => Ok(Self::Str),
-            b'a' => Ok(Self::Ascii),
-            b'r' => Ok(Self::Repr),
-            b => Err(b as usize),
-        }
+        u8::try_from(b)
+            .ok()
+            .and_then(|b| Self::try_from(b).ok())
+            .ok_or(b)
     }
 }
 
 /// The kind of Raise that occurred.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
+#[repr(u8)]
 pub enum RaiseKind {
     Reraise,
     Raise,
     RaiseCause,
 }
+enum_oparg!(RaiseKind);
 
 pub type NameIdx = u32;
 
 /// A Single bytecode instruction.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub enum Instruction {
     /// Importing by name
     ImportName {
-        idx: NameIdx,
+        idx: Arg<NameIdx>,
     },
     /// Importing without name
     ImportNameless,
@@ -174,52 +351,52 @@ pub enum Instruction {
     ImportStar,
     /// from ... import ...
     ImportFrom {
-        idx: NameIdx,
+        idx: Arg<NameIdx>,
     },
-    LoadFast(NameIdx),
-    LoadNameAny(NameIdx),
-    LoadGlobal(NameIdx),
-    LoadDeref(NameIdx),
-    LoadClassDeref(NameIdx),
-    StoreFast(NameIdx),
-    StoreLocal(NameIdx),
-    StoreGlobal(NameIdx),
-    StoreDeref(NameIdx),
-    DeleteFast(NameIdx),
-    DeleteLocal(NameIdx),
-    DeleteGlobal(NameIdx),
-    DeleteDeref(NameIdx),
-    LoadClosure(NameIdx),
+    LoadFast(Arg<NameIdx>),
+    LoadNameAny(Arg<NameIdx>),
+    LoadGlobal(Arg<NameIdx>),
+    LoadDeref(Arg<NameIdx>),
+    LoadClassDeref(Arg<NameIdx>),
+    StoreFast(Arg<NameIdx>),
+    StoreLocal(Arg<NameIdx>),
+    StoreGlobal(Arg<NameIdx>),
+    StoreDeref(Arg<NameIdx>),
+    DeleteFast(Arg<NameIdx>),
+    DeleteLocal(Arg<NameIdx>),
+    DeleteGlobal(Arg<NameIdx>),
+    DeleteDeref(Arg<NameIdx>),
+    LoadClosure(Arg<NameIdx>),
     Subscript,
     StoreSubscript,
     DeleteSubscript,
     StoreAttr {
-        idx: NameIdx,
+        idx: Arg<NameIdx>,
     },
     DeleteAttr {
-        idx: NameIdx,
+        idx: Arg<NameIdx>,
     },
     LoadConst {
         /// index into constants vec
-        idx: u32,
+        idx: Arg<u32>,
     },
     UnaryOperation {
-        op: UnaryOperator,
+        op: Arg<UnaryOperator>,
     },
     BinaryOperation {
-        op: BinaryOperator,
+        op: Arg<BinaryOperator>,
     },
     BinaryOperationInplace {
-        op: BinaryOperator,
+        op: Arg<BinaryOperator>,
     },
     LoadAttr {
-        idx: NameIdx,
+        idx: Arg<NameIdx>,
     },
     TestOperation {
-        op: TestOperator,
+        op: Arg<TestOperator>,
     },
     CompareOperation {
-        op: ComparisonOperator,
+        op: Arg<ComparisonOperator>,
     },
     Pop,
     Rotate2,
@@ -228,71 +405,69 @@ pub enum Instruction {
     Duplicate2,
     GetIter,
     Continue {
-        target: Label,
+        target: Arg<Label>,
     },
     Break {
-        target: Label,
+        target: Arg<Label>,
     },
     Jump {
-        target: Label,
+        target: Arg<Label>,
     },
     /// Pop the top of the stack, and jump if this value is true.
     JumpIfTrue {
-        target: Label,
+        target: Arg<Label>,
     },
     /// Pop the top of the stack, and jump if this value is false.
     JumpIfFalse {
-        target: Label,
+        target: Arg<Label>,
     },
     /// Peek at the top of the stack, and jump if this value is true.
     /// Otherwise, pop top of stack.
     JumpIfTrueOrPop {
-        target: Label,
+        target: Arg<Label>,
     },
     /// Peek at the top of the stack, and jump if this value is false.
     /// Otherwise, pop top of stack.
     JumpIfFalseOrPop {
-        target: Label,
+        target: Arg<Label>,
     },
-    MakeFunction(MakeFunctionFlags),
+    MakeFunction(Arg<MakeFunctionFlags>),
     CallFunctionPositional {
-        nargs: u32,
+        nargs: Arg<u32>,
     },
     CallFunctionKeyword {
-        nargs: u32,
+        nargs: Arg<u32>,
     },
     CallFunctionEx {
-        has_kwargs: bool,
+        has_kwargs: Arg<bool>,
     },
     LoadMethod {
-        idx: NameIdx,
+        idx: Arg<NameIdx>,
     },
     CallMethodPositional {
-        nargs: u32,
+        nargs: Arg<u32>,
     },
     CallMethodKeyword {
-        nargs: u32,
+        nargs: Arg<u32>,
     },
     CallMethodEx {
-        has_kwargs: bool,
+        has_kwargs: Arg<bool>,
     },
     ForIter {
-        target: Label,
+        target: Arg<Label>,
     },
     ReturnValue,
     YieldValue,
     YieldFrom,
     SetupAnnotation,
-    SetupLoop {
-        break_target: Label,
-    },
+    SetupLoop,
 
     /// Setup a finally handler, which will be called whenever one of this events occurs:
     /// - the block is popped
     /// - the function returns
     /// - an exception is returned
     SetupFinally {
-        handler: Label,
+        handler: Arg<Label>,
     },
 
     /// Enter a finally block, without returning, excepting, just because we are there.
@@ -307,78 +482,99 @@ pub enum Instruction {
     EndFinally,
 
     SetupExcept {
-        handler: Label,
+        handler: Arg<Label>,
     },
     SetupWith {
-        end: Label,
+        end: Arg<Label>,
     },
     WithCleanupStart,
     WithCleanupFinish,
     PopBlock,
     Raise {
-        kind: RaiseKind,
+        kind: Arg<RaiseKind>,
     },
     BuildString {
-        size: u32,
+        size: Arg<u32>,
     },
     BuildTuple {
-        unpack: bool,
-        size: u32,
+        size: Arg<u32>,
+    },
+    BuildTupleUnpack {
+        size: Arg<u32>,
     },
     BuildList {
-        unpack: bool,
-        size: u32,
+        size: Arg<u32>,
+    },
+    BuildListUnpack {
+        size: Arg<u32>,
     },
     BuildSet {
-        unpack: bool,
-        size: u32,
+        size: Arg<u32>,
+    },
+    BuildSetUnpack {
+        size: Arg<u32>,
     },
     BuildMap {
-        unpack: bool,
-        for_call: bool,
-        size: u32,
+        size: Arg<u32>,
+    },
+    BuildMapForCall {
+        size: Arg<u32>,
     },
     DictUpdate,
     BuildSlice {
         /// whether build a slice with a third step argument
-        step: bool,
+        step: Arg<bool>,
     },
     ListAppend {
-        i: u32,
+        i: Arg<u32>,
     },
     SetAdd {
-        i: u32,
+        i: Arg<u32>,
     },
     MapAdd {
-        i: u32,
+        i: Arg<u32>,
     },
 
     PrintExpr,
     LoadBuildClass,
     UnpackSequence {
-        size: u32,
+        size: Arg<u32>,
     },
     UnpackEx {
-        before: u8,
-        after: u8,
+        args: Arg<UnpackExArgs>,
     },
     FormatValue {
-        conversion: ConversionFlag,
+        conversion: Arg<ConversionFlag>,
     },
     PopException,
     Reverse {
-        amount: u32,
+        amount: Arg<u32>,
     },
     GetAwaitable,
     BeforeAsyncWith,
     SetupAsyncWith {
-        end: Label,
+        end: Arg<Label>,
     },
     GetAIter,
     GetANext,
     EndAsyncFor,
+    ExtendedArg,
+}
+const _: () = assert!(mem::size_of::<Instruction>() == 1);
+
+#[derive(Copy, Clone, Serialize, Deserialize)]
+pub struct CodeUnit {
+    pub op: Instruction,
+    pub arg: OpArgByte,
+}
+
+const _: () = assert!(mem::size_of::<CodeUnit>() == 2);
+
+impl CodeUnit {
+    pub fn new(op: Instruction, arg: OpArgByte) -> Self {
+        Self { op, arg }
+    }
 }
-static_assertions::assert_eq_size!(Instruction, u64);
 
 use self::Instruction::*;
 
@@ -391,6 +587,16 @@ bitflags! {
         const DEFAULTS = 0x08;
     }
 }
+impl OpArgType for MakeFunctionFlags {
+    #[inline(always)]
+    fn from_oparg(x: u32) -> Option<Self> {
+        Some(unsafe { MakeFunctionFlags::from_bits_unchecked(x as u8) })
+    }
+    #[inline(always)]
+    fn to_oparg(self) -> u32 {
+        self.bits().into()
+    }
+}
 
 /// A Constant (which usually encapsulates data within it)
 ///
@@ -443,7 +649,7 @@ impl Eq for ConstantData {}
 impl hash::Hash for ConstantData {
     fn hash<H: hash::Hasher>(&self, state: &mut H) {
         use ConstantData::*;
-        std::mem::discriminant(self).hash(state);
+        mem::discriminant(self).hash(state);
         match self {
             Integer { value } => value.hash(state),
             Float { value } => value.to_bits().hash(state),
@@ -536,7 +742,8 @@ impl<C: Constant> BorrowedConstant<'_, C> {
 }
 
 /// The possible comparison operators
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
+#[repr(u8)]
 pub enum ComparisonOperator {
     // be intentional with bits so that we can do eval_ord with just a bitwise and
     // bits: | Equal | Greater | Less |
@@ -547,8 +754,10 @@ pub enum ComparisonOperator {
     LessOrEqual = 0b101,
     GreaterOrEqual = 0b110,
 }
+enum_oparg!(ComparisonOperator);
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
+#[repr(u8)]
 pub enum TestOperator {
     In,
     NotIn,
@@ -557,16 +766,18 @@ pub enum TestOperator {
     /// two exceptions that match?
     ExceptionMatch,
 }
+enum_oparg!(TestOperator);
 
 /// The possible Binary operators
 /// # Examples
 ///
-/// ```
+/// ```ignore
 /// use rustpython_compiler_core::Instruction::BinaryOperation;
 /// use rustpython_compiler_core::BinaryOperator::Add;
 /// let op = BinaryOperation {op: Add};
 /// ```
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
+#[repr(u8)]
 pub enum BinaryOperator {
     Power,
     Multiply,
@@ -582,15 +793,41 @@ pub enum BinaryOperator {
     Xor,
     Or,
 }
+enum_oparg!(BinaryOperator);
 
 /// The possible unary operators
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
+#[repr(u8)]
 pub enum UnaryOperator {
     Not,
     Invert,
     Minus,
     Plus,
 }
+enum_oparg!(UnaryOperator);
+
+#[derive(Copy, Clone)]
+pub struct UnpackExArgs {
+    pub before: u8,
+    pub after: u8,
+}
+
+impl OpArgType for UnpackExArgs {
+    #[inline(always)]
+    fn from_oparg(x: u32) -> Option<Self> {
+        let [before, after, ..] = x.to_le_bytes();
+        Some(Self { before, after })
+    }
+    #[inline(always)]
+    fn to_oparg(self) -> u32 {
+        u32::from_le_bytes([self.before, self.after, 0, 0])
+    }
+}
+impl fmt::Display for UnpackExArgs {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "before: {}, after: {}", self.before, self.after)
+    }
+}
 
 /*
 Maintain a stack of blocks on the VM.
@@ -662,9 +899,11 @@ impl<C: Constant> CodeObject<C> {
     /// Return the labels targeted by the instructions of this CodeObject
     pub fn label_targets(&self) -> BTreeSet<Label> {
         let mut label_targets = BTreeSet::new();
+        let mut arg_state = OpArgState::default();
         for instruction in &*self.instructions {
+            let (instruction, arg) = arg_state.get(*instruction);
             if let Some(l) = instruction.label_arg() {
-                label_targets.insert(*l);
+                label_targets.insert(l.get(arg));
             }
         }
         label_targets
@@ -680,7 +919,9 @@ impl<C: Constant> CodeObject<C> {
         let line_digits = (3).max(self.locations.last().unwrap().row.to_string().len());
         let offset_digits = (4).max(self.instructions.len().to_string().len());
         let mut last_line = u32::MAX;
-        for (offset, instruction) in self.instructions.iter().enumerate() {
+        let mut arg_state = OpArgState::default();
+        for (offset, &instruction) in self.instructions.iter().enumerate() {
+            let (instruction, arg) = arg_state.get(instruction);
             // optional line number
             let line = self.locations[offset].row;
             if line != last_line {
@@ -710,16 +951,8 @@ impl<C: Constant> CodeObject<C> {
             write!(f, "{arrow} {offset:offset_digits$} ")?;
 
             // instruction
-            instruction.fmt_dis(
-                f,
-                &self.constants,
-                &self.names,
-                &self.varnames,
-                &self.cellvars,
-                &self.freevars,
-                expand_codeobjects,
-                level,
-            )?;
+            instruction.fmt_dis(arg, f, self, expand_codeobjects, 21, level)?;
+            writeln!(f)?;
         }
         Ok(())
     }
@@ -863,7 +1096,7 @@ impl<C: Constant> fmt::Display for CodeObject<C> {
 impl Instruction {
     /// Gets the label stored inside this instruction, if it exists
     #[inline]
-    pub fn label_arg(&self) -> Option<&Label> {
+    pub fn label_arg(&self) -> Option<Arg<Label>> {
         match self {
             Jump { target: l }
             | JumpIfTrue { target: l }
@@ -875,28 +1108,8 @@ impl Instruction {
             | SetupExcept { handler: l }
             | SetupWith { end: l }
             | SetupAsyncWith { end: l }
-            | SetupLoop { break_target: l }
-            | Continue { target: l } => Some(l),
-            _ => None,
-        }
-    }
-
-    /// Gets a mutable reference to the label stored inside this instruction, if it exists
-    #[inline]
-    pub fn label_arg_mut(&mut self) -> Option<&mut Label> {
-        match self {
-            Jump { target: l }
-            | JumpIfTrue { target: l }
-            | JumpIfFalse { target: l }
-            | JumpIfTrueOrPop { target: l }
-            | JumpIfFalseOrPop { target: l }
-            | ForIter { target: l }
-            | SetupFinally { handler: l }
-            | SetupExcept { handler: l }
-            | SetupWith { end: l }
-            | SetupAsyncWith { end: l }
-            | SetupLoop { break_target: l }
-            | Continue { target: l } => Some(l),
+            | Break { target: l }
+            | Continue { target: l } => Some(*l),
             _ => None,
         }
     }
@@ -906,9 +1119,8 @@ impl Instruction {
     /// # Examples
     ///
     /// ```
-    /// use rustpython_compiler_core::{Instruction, Label};
-    /// let label = Label(0xF);
-    /// let jump_inst = Instruction::Jump {target: label};
+    /// use rustpython_compiler_core::{Arg, Instruction};
+    /// let jump_inst = Instruction::Jump { target: Arg::marker() };
     /// assert!(jump_inst.unconditional_branch())
     /// ```
     pub fn unconditional_branch(&self) -> bool {
@@ -923,14 +1135,16 @@ impl Instruction {
     /// # Examples
     ///
     /// ```
-    /// use rustpython_compiler_core::{Instruction, Label, UnaryOperator};
-    /// let jump_instruction = Instruction::Jump {target: Label(0xF)};
-    /// let invert_instruction = Instruction::UnaryOperation {op: UnaryOperator::Invert};
-    /// assert_eq!(jump_instruction.stack_effect(true), 0);
-    /// assert_eq!(invert_instruction.stack_effect(false), 0);
+    /// use rustpython_compiler_core::{Arg, Instruction, Label, UnaryOperator};
+    /// let (target, jump_arg) = Arg::new(Label(0xF));
+    /// let jump_instruction = Instruction::Jump { target };
+    /// let (op, invert_arg) = Arg::new(UnaryOperator::Invert);
+    /// let invert_instruction = Instruction::UnaryOperation { op };
+    /// assert_eq!(jump_instruction.stack_effect(jump_arg, true), 0);
+    /// assert_eq!(invert_instruction.stack_effect(invert_arg, false), 0);
     /// ```
     ///
-    pub fn stack_effect(&self, jump: bool) -> i32 {
+    pub fn stack_effect(&self, arg: OpArg, jump: bool) -> i32 {
         match self {
             ImportName { .. } | ImportNameless => -1,
             ImportStar => -1,
@@ -968,18 +1182,19 @@ impl Instruction {
                 }
             }
             MakeFunction(flags) => {
+                let flags = flags.get(arg);
                 -2 - flags.contains(MakeFunctionFlags::CLOSURE) as i32
                     - flags.contains(MakeFunctionFlags::ANNOTATIONS) as i32
                     - flags.contains(MakeFunctionFlags::KW_ONLY_DEFAULTS) as i32
                     - flags.contains(MakeFunctionFlags::DEFAULTS) as i32
                     + 1
             }
-            CallFunctionPositional { nargs } => -(*nargs as i32) - 1 + 1,
-            CallMethodPositional { nargs } => -(*nargs as i32) - 3 + 1,
-            CallFunctionKeyword { nargs } => -1 - (*nargs as i32) - 1 + 1,
-            CallMethodKeyword { nargs } => -1 - (*nargs as i32) - 3 + 1,
-            CallFunctionEx { has_kwargs } => -1 - (*has_kwargs as i32) - 1 + 1,
-            CallMethodEx { has_kwargs } => -1 - (*has_kwargs as i32) - 3 + 1,
+            CallFunctionPositional { nargs } => -(nargs.get(arg) as i32) - 1 + 1,
+            CallMethodPositional { nargs } => -(nargs.get(arg) as i32) - 3 + 1,
+            CallFunctionKeyword { nargs } => -1 - (nargs.get(arg) as i32) - 1 + 1,
+            CallMethodKeyword { nargs } => -1 - (nargs.get(arg) as i32) - 3 + 1,
+            CallFunctionEx { has_kwargs } => -1 - (has_kwargs.get(arg) as i32) - 1 + 1,
+            CallMethodEx { has_kwargs } => -1 - (has_kwargs.get(arg) as i32) - 3 + 1,
             LoadMethod { .. } => -1 + 3,
             ForIter { .. } => {
                 if jump {
@@ -991,33 +1206,39 @@ impl Instruction {
             ReturnValue => -1,
             YieldValue => 0,
             YieldFrom => -1,
-            SetupAnnotation
-            | SetupLoop { .. }
-            | SetupFinally { .. }
-            | EnterFinally
-            | EndFinally => 0,
+            SetupAnnotation | SetupLoop | SetupFinally { .. } | EnterFinally | EndFinally => 0,
             SetupExcept { .. } => jump as i32,
             SetupWith { .. } => (!jump) as i32,
             WithCleanupStart => 0,
             WithCleanupFinish => -1,
             PopBlock => 0,
-            Raise { kind } => -(*kind as u8 as i32),
+            Raise { kind } => -(kind.get(arg) as u8 as i32),
             BuildString { size }
             | BuildTuple { size, .. }
+            | BuildTupleUnpack { size, .. }
             | BuildList { size, .. }
-            | BuildSet { size, .. } => -(*size as i32) + 1,
-            BuildMap { unpack, size, .. } => {
-                let nargs = if *unpack { *size } else { *size * 2 };
+            | BuildListUnpack { size, .. }
+            | BuildSet { size, .. }
+            | BuildSetUnpack { size, .. } => -(size.get(arg) as i32) + 1,
+            BuildMap { size } => {
+                let nargs = size.get(arg) * 2;
+                -(nargs as i32) + 1
+            }
+            BuildMapForCall { size } => {
+                let nargs = size.get(arg);
                 -(nargs as i32) + 1
             }
             DictUpdate => -1,
-            BuildSlice { step } => -2 - (*step as i32) + 1,
+            BuildSlice { step } => -2 - (step.get(arg) as i32) + 1,
             ListAppend { .. } | SetAdd { .. } => -1,
             MapAdd { .. } => -2,
             PrintExpr => -1,
             LoadBuildClass => 1,
-            UnpackSequence { size } => -1 + *size as i32,
-            UnpackEx { before, after } => -1 + *before as i32 + 1 + *after as i32,
+            UnpackSequence { size } => -1 + size.get(arg) as i32,
+            UnpackEx { args } => {
+                let UnpackExArgs { before, after } = args.get(arg);
+                -1 + before as i32 + 1 + after as i32
+            }
             FormatValue { .. } => -1,
             PopException => 0,
             Reverse { .. } => 0,
@@ -1033,99 +1254,104 @@ impl Instruction {
             GetAIter => 0,
             GetANext => 1,
             EndAsyncFor => -2,
+            ExtendedArg => 0,
         }
     }
 
+    pub fn display<'a>(
+        &'a self,
+        arg: OpArg,
+        ctx: &'a impl InstrDisplayContext,
+    ) -> impl fmt::Display + 'a {
+        struct FmtFn<F>(F);
+        impl<F: Fn(&mut fmt::Formatter) -> fmt::Result> fmt::Display for FmtFn<F> {
+            fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+                (self.0)(f)
+            }
+        }
+        FmtFn(move |f: &mut fmt::Formatter| self.fmt_dis(arg, f, ctx, false, 0, 0))
+    }
+
     #[allow(clippy::too_many_arguments)]
-    fn fmt_dis<C: Constant>(
+    fn fmt_dis(
         &self,
+        arg: OpArg,
         f: &mut fmt::Formatter,
-        constants: &[C],
-        names: &[C::Name],
-        varnames: &[C::Name],
-        cellvars: &[C::Name],
-        freevars: &[C::Name],
+        ctx: &impl InstrDisplayContext,
         expand_codeobjects: bool,
+        pad: usize,
         level: usize,
     ) -> fmt::Result {
         macro_rules! w {
             ($variant:ident) => {
-                writeln!(f, stringify!($variant))
+                write!(f, stringify!($variant))
             };
-            ($variant:ident, $var:expr) => {
-                writeln!(f, "{:20} ({})", stringify!($variant), $var)
+            ($variant:ident, $map:ident = $argmarker:expr) => {{
+                let arg = $argmarker.get(arg);
+                write!(f, "{:pad$}({}, {})", stringify!($variant), arg, $map(arg))
+            }};
+            ($variant:ident, $argmarker:expr) => {
+                write!(f, "{:pad$}({})", stringify!($variant), $argmarker.get(arg))
             };
-            ($variant:ident, $var1:expr, $var2:expr) => {
-                writeln!(f, "{:20} ({}, {})", stringify!($variant), $var1, $var2)
-            };
-            ($variant:ident, $var1:expr, $var2:expr, $var3:expr) => {
-                writeln!(
+            ($variant:ident, ?$argmarker:expr) => {
+                write!(
                     f,
-                    "{:20} ({}, {}, {})",
+                    "{:pad$}({:?})",
                     stringify!($variant),
-                    $var1,
-                    $var2,
-                    $var3
+                    $argmarker.get(arg)
                 )
             };
         }
 
-        let varname = |i: u32| varnames[i as usize].as_ref();
-        let name = |i: u32| names[i as usize].as_ref();
-        let cellname = |i: u32| {
-            cellvars
-                .get(i as usize)
-                .unwrap_or_else(|| &freevars[i as usize - cellvars.len()])
-                .as_ref()
-        };
+        let varname = |i: u32| ctx.get_varname(i as usize);
+        let name = |i: u32| ctx.get_name(i as usize);
+        let cellname = |i: u32| ctx.get_cellname(i as usize);
 
         match self {
-            ImportName { idx } => w!(ImportName, name(*idx)),
+            ImportName { idx } => w!(ImportName, name = idx),
             ImportNameless => w!(ImportNameless),
             ImportStar => w!(ImportStar),
-            ImportFrom { idx } => w!(ImportFrom, name(*idx)),
-            LoadFast(idx) => w!(LoadFast, *idx, varname(*idx)),
-            LoadNameAny(idx) => w!(LoadNameAny, *idx, name(*idx)),
-            LoadGlobal(idx) => w!(LoadGlobal, *idx, name(*idx)),
-            LoadDeref(idx) => w!(LoadDeref, *idx, cellname(*idx)),
-            LoadClassDeref(idx) => w!(LoadClassDeref, *idx, cellname(*idx)),
-            StoreFast(idx) => w!(StoreFast, *idx, varname(*idx)),
-            StoreLocal(idx) => w!(StoreLocal, *idx, name(*idx)),
-            StoreGlobal(idx) => w!(StoreGlobal, *idx, name(*idx)),
-            StoreDeref(idx) => w!(StoreDeref, *idx, cellname(*idx)),
-            DeleteFast(idx) => w!(DeleteFast, *idx, varname(*idx)),
-            DeleteLocal(idx) => w!(DeleteLocal, *idx, name(*idx)),
-            DeleteGlobal(idx) => w!(DeleteGlobal, *idx, name(*idx)),
-            DeleteDeref(idx) => w!(DeleteDeref, *idx, cellname(*idx)),
-            LoadClosure(i) => w!(LoadClosure, *i, cellname(*i)),
+            ImportFrom { idx } => w!(ImportFrom, name = idx),
+            LoadFast(idx) => w!(LoadFast, varname = idx),
+            LoadNameAny(idx) => w!(LoadNameAny, name = idx),
+            LoadGlobal(idx) => w!(LoadGlobal, name = idx),
+            LoadDeref(idx) => w!(LoadDeref, cellname = idx),
+            LoadClassDeref(idx) => w!(LoadClassDeref, cellname = idx),
+            StoreFast(idx) => w!(StoreFast, varname = idx),
+            StoreLocal(idx) => w!(StoreLocal, name = idx),
+            StoreGlobal(idx) => w!(StoreGlobal, name = idx),
+            StoreDeref(idx) => w!(StoreDeref, cellname = idx),
+            DeleteFast(idx) => w!(DeleteFast, varname = idx),
+            DeleteLocal(idx) => w!(DeleteLocal, name = idx),
+            DeleteGlobal(idx) => w!(DeleteGlobal, name = idx),
+            DeleteDeref(idx) => w!(DeleteDeref, cellname = idx),
+            LoadClosure(i) => w!(LoadClosure, cellname = i),
             Subscript => w!(Subscript),
             StoreSubscript => w!(StoreSubscript),
             DeleteSubscript => w!(DeleteSubscript),
-            StoreAttr { idx } => w!(StoreAttr, name(*idx)),
-            DeleteAttr { idx } => w!(DeleteAttr, name(*idx)),
+            StoreAttr { idx } => w!(StoreAttr, name = idx),
+            DeleteAttr { idx } => w!(DeleteAttr, name = idx),
             LoadConst { idx } => {
-                let value = &constants[*idx as usize];
+                let value = ctx.get_constant(idx.get(arg) as usize);
                 match value.borrow_constant() {
                     BorrowedConstant::Code { code } if expand_codeobjects => {
-                        writeln!(f, "{:20} ({:?}):", "LoadConst", code)?;
+                        write!(f, "{:pad$}({:?}):", "LoadConst", code)?;
                         code.display_inner(f, true, level + 1)?;
                         Ok(())
                     }
                     c => {
-                        write!(f, "{:20} (", "LoadConst")?;
+                        write!(f, "{:pad$}(", "LoadConst")?;
                         c.fmt_display(f)?;
-                        writeln!(f, ")")
+                        write!(f, ")")
                     }
                 }
             }
-            UnaryOperation { op } => w!(UnaryOperation, format_args!("{op:?}")),
-            BinaryOperation { op } => w!(BinaryOperation, format_args!("{op:?}")),
-            BinaryOperationInplace { op } => {
-                w!(BinaryOperationInplace, format_args!("{op:?}"))
-            }
-            LoadAttr { idx } => w!(LoadAttr, name(*idx)),
-            TestOperation { op } => w!(TestOperation, format_args!("{op:?}")),
-            CompareOperation { op } => w!(CompareOperation, format_args!("{op:?}")),
+            UnaryOperation { op } => w!(UnaryOperation, ?op),
+            BinaryOperation { op } => w!(BinaryOperation, ?op),
+            BinaryOperationInplace { op } => w!(BinaryOperationInplace, ?op),
+            LoadAttr { idx } => w!(LoadAttr, name = idx),
+            TestOperation { op } => w!(TestOperation, ?op),
+            CompareOperation { op } => w!(CompareOperation, ?op),
             Pop => w!(Pop),
             Rotate2 => w!(Rotate2),
             Rotate3 => w!(Rotate3),
@@ -1139,11 +1365,11 @@ impl Instruction {
             JumpIfFalse { target } => w!(JumpIfFalse, target),
             JumpIfTrueOrPop { target } => w!(JumpIfTrueOrPop, target),
             JumpIfFalseOrPop { target } => w!(JumpIfFalseOrPop, target),
-            MakeFunction(flags) => w!(MakeFunction, format_args!("{flags:?}")),
+            MakeFunction(flags) => w!(MakeFunction, ?flags),
             CallFunctionPositional { nargs } => w!(CallFunctionPositional, nargs),
             CallFunctionKeyword { nargs } => w!(CallFunctionKeyword, nargs),
             CallFunctionEx { has_kwargs } => w!(CallFunctionEx, has_kwargs),
-            LoadMethod { idx } => w!(LoadMethod, name(*idx)),
+            LoadMethod { idx } => w!(LoadMethod, name = idx),
             CallMethodPositional { nargs } => w!(CallMethodPositional, nargs),
             CallMethodKeyword { nargs } => w!(CallMethodKeyword, nargs),
             CallMethodEx { has_kwargs } => w!(CallMethodEx, has_kwargs),
@@ -1152,7 +1378,7 @@ impl Instruction {
             YieldValue => w!(YieldValue),
             YieldFrom => w!(YieldFrom),
             SetupAnnotation => w!(SetupAnnotation),
-            SetupLoop { break_target } => w!(SetupLoop, break_target),
+            SetupLoop => w!(SetupLoop),
             SetupExcept { handler } => w!(SetupExcept, handler),
             SetupFinally { handler } => w!(SetupFinally, handler),
             EnterFinally => w!(EnterFinally),
@@ -1163,16 +1389,16 @@ impl Instruction {
             BeforeAsyncWith => w!(BeforeAsyncWith),
             SetupAsyncWith { end } => w!(SetupAsyncWith, end),
             PopBlock => w!(PopBlock),
-            Raise { kind } => w!(Raise, format_args!("{kind:?}")),
+            Raise { kind } => w!(Raise, ?kind),
             BuildString { size } => w!(BuildString, size),
-            BuildTuple { size, unpack } => w!(BuildTuple, size, unpack),
-            BuildList { size, unpack } => w!(BuildList, size, unpack),
-            BuildSet { size, unpack } => w!(BuildSet, size, unpack),
-            BuildMap {
-                size,
-                unpack,
-                for_call,
-            } => w!(BuildMap, size, unpack, for_call),
+            BuildTuple { size } => w!(BuildTuple, size),
+            BuildTupleUnpack { size } => w!(BuildTupleUnpack, size),
+            BuildList { size } => w!(BuildList, size),
+            BuildListUnpack { size } => w!(BuildListUnpack, size),
+            BuildSet { size } => w!(BuildSet, size),
+            BuildSetUnpack { size } => w!(BuildSetUnpack, size),
+            BuildMap { size } => w!(BuildMap, size),
+            BuildMapForCall { size } => w!(BuildMap, size),
             DictUpdate => w!(DictUpdate),
             BuildSlice { step } => w!(BuildSlice, step),
             ListAppend { i } => w!(ListAppend, i),
@@ -1181,18 +1407,46 @@ impl Instruction {
             PrintExpr => w!(PrintExpr),
             LoadBuildClass => w!(LoadBuildClass),
             UnpackSequence { size } => w!(UnpackSequence, size),
-            UnpackEx { before, after } => w!(UnpackEx, before, after),
-            FormatValue { conversion } => w!(FormatValue, format_args!("{conversion:?}")),
+            UnpackEx { args } => w!(UnpackEx, args),
+            FormatValue { conversion } => w!(FormatValue, ?conversion),
             PopException => w!(PopException),
             Reverse { amount } => w!(Reverse, amount),
             GetAwaitable => w!(GetAwaitable),
             GetAIter => w!(GetAIter),
             GetANext => w!(GetANext),
             EndAsyncFor => w!(EndAsyncFor),
+            ExtendedArg => w!(ExtendedArg, Arg::<u32>::marker()),
         }
     }
 }
 
+pub trait InstrDisplayContext {
+    type Constant: Constant;
+    fn get_constant(&self, i: usize) -> &Self::Constant;
+    fn get_name(&self, i: usize) -> &str;
+    fn get_varname(&self, i: usize) -> &str;
+    fn get_cellname(&self, i: usize) -> &str;
+}
+
+impl<C: Constant> InstrDisplayContext for CodeObject<C> {
+    type Constant = C;
+    fn get_constant(&self, i: usize) -> &C {
+        &self.constants[i]
+    }
+    fn get_name(&self, i: usize) -> &str {
+        self.names[i].as_ref()
+    }
+    fn get_varname(&self, i: usize) -> &str {
+        self.varnames[i].as_ref()
+    }
+    fn get_cellname(&self, i: usize) -> &str {
+        self.cellvars
+            .get(i)
+            .unwrap_or_else(|| &self.freevars[i - self.cellvars.len()])
+            .as_ref()
+    }
+}
+
 impl fmt::Display for ConstantData {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         self.borrow_constant().fmt_display(f)
diff --git a/src/lib.rs b/src/lib.rs
index 1a8d6f32..657d0fe7 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -6,8 +6,7 @@ use rustpython_parser::{
 };
 
 pub use rustpython_codegen::compile::CompileOpts;
-pub use rustpython_compiler_core::CodeObject;
-pub use rustpython_compiler_core::{BaseError as CompileErrorBody, Mode};
+pub use rustpython_compiler_core::{BaseError as CompileErrorBody, CodeObject, Mode};
 
 #[derive(Debug, thiserror::Error)]
 pub enum CompileErrorType {
@@ -29,7 +28,7 @@ pub fn compile(
     source: &str,
     mode: compile::Mode,
     source_path: String,
-    opts: compile::CompileOpts,
+    opts: CompileOpts,
 ) -> Result<CodeObject, CompileError> {
     let mut ast = match parser::parse(source, mode.into(), &source_path) {
         Ok(x) => x,

From bb2abd1cbc237d5aa6d18e83af7b7ebf016ad7be Mon Sep 17 00:00:00 2001
From: Ran Benita <ran@unusedvar.com>
Date: Thu, 12 Jan 2023 22:34:04 +0200
Subject: [PATCH 387/459] Fixup parse_tokens after "Add NonLogicalNewline
 token"

I only updated `parse()` to ignore `NonLogicalNewline`, didn't notice
it's also needed in `parse_tokens()`.
---
 parser/src/parser.rs | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 4d9b52bb..fb181559 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -93,14 +93,7 @@ pub fn parse_located(
     location: Location,
 ) -> Result<ast::Mod, ParseError> {
     let lxr = lexer::make_tokenizer_located(source, location);
-    let marker_token = (Default::default(), mode.to_marker(), Default::default());
-    let tokenizer = iter::once(Ok(marker_token))
-        .chain(lxr)
-        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
-
-    python::TopParser::new()
-        .parse(tokenizer)
-        .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
+    parse_tokens(lxr, mode, source_path)
 }
 
 // Parse a given token iterator.
@@ -112,7 +105,7 @@ pub fn parse_tokens(
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
     let tokenizer = iter::once(Ok(marker_token))
         .chain(lxr)
-        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment(_)));
+        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
 
     python::TopParser::new()
         .parse(tokenizer)

From 52d2e25601fd5a036d58859ca48911fa2b3d980f Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 15 Jan 2023 13:01:59 +0900
Subject: [PATCH 388/459] Fix dict spreading in dict literal

---
 ast/src/ast_gen.rs                            |  2 +-
 parser/python.lalrpop                         | 72 ++++++++---------
 parser/src/parser.rs                          |  6 ++
 ...parser__tests__dict_containing_spread.snap | 81 +++++++++++++++++++
 4 files changed, 124 insertions(+), 37 deletions(-)
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__dict_containing_spread.snap

diff --git a/ast/src/ast_gen.rs b/ast/src/ast_gen.rs
index 241ebe13..32dd89c2 100644
--- a/ast/src/ast_gen.rs
+++ b/ast/src/ast_gen.rs
@@ -195,7 +195,7 @@ pub enum ExprKind<U = ()> {
         orelse: Box<Expr<U>>,
     },
     Dict {
-        keys: Vec<Expr<U>>,
+        keys: Vec<Option<Expr<U>>>,
         values: Vec<Expr<U>>,
     },
     Set {
diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 90f4ce25..70079057 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -1138,42 +1138,42 @@ Atom<Goal>: ast::Expr = {
     <location:@L> "{" <e:DictLiteralValues?> "}" <end_location:@R> => {
         let pairs = e.unwrap_or_default();
 
-        let (keys, values) = match pairs.iter().position(|(k,_)| k.is_none()) {
-            Some(unpack_idx) => {
-                let mut pairs = pairs;
-                let (keys, mut values): (_, Vec<_>) = pairs.drain(..unpack_idx).map(|(k, v)| (*k.unwrap(), v)).unzip();
-
-                fn build_map(items: &mut Vec<(ast::Expr, ast::Expr)>) -> ast::Expr {
-                    let location = items[0].0.location;
-                    let end_location = items[0].0.end_location;
-                    let (keys, values) = items.drain(..).unzip();
-                    ast::Expr {
-                        location,
-                        end_location,
-                        custom: (),
-                        node: ast::ExprKind::Dict { keys, values }
-                    }
-                }
-
-                let mut items = Vec::new();
-                for (key, value) in pairs.into_iter() {
-                    if let Some(key) = key {
-                        items.push((*key, value));
-                        continue;
-                    }
-                    if !items.is_empty() {
-                        values.push(build_map(&mut items));
-                    }
-                    values.push(value);
-                }
-                if !items.is_empty() {
-                    values.push(build_map(&mut items));
-                }
-                (keys, values)
-            },
-            None => pairs.into_iter().map(|(k, v)| (*k.unwrap(), v)).unzip()
-        };
-
+        // let (keys, values) = match pairs.iter().position(|(k,_)| k.is_none()) {
+        //     Some(unpack_idx) => {
+        //         let mut pairs = pairs;
+        //         let (keys, mut values): (_, Vec<_>) = pairs.drain(..unpack_idx).map(|(k, v)| (*k.unwrap(), v)).unzip();
+        //
+        //         fn build_map(items: &mut Vec<(ast::Expr, ast::Expr)>) -> ast::Expr {
+        //             let location = items[0].0.location;
+        //             let end_location = items[0].0.end_location;
+        //             let (keys, values) = items.drain(..).unzip();
+        //             ast::Expr {
+        //                 location,
+        //                 end_location,
+        //                 custom: (),
+        //                 node: ast::ExprKind::Dict { keys, values }
+        //             }
+        //         }
+        //
+        //         let mut items = Vec::new();
+        //         for (key, value) in pairs.into_iter() {
+        //             if let Some(key) = key {
+        //                 items.push((*key, value));
+        //                 continue;
+        //             }
+        //             if !items.is_empty() {
+        //                 values.push(build_map(&mut items));
+        //             }
+        //             values.push(value);
+        //         }
+        //         if !items.is_empty() {
+        //             values.push(build_map(&mut items));
+        //         }
+        //         (keys, values)
+        //     },
+        //     None => pairs.into_iter().map(|(k, v)| (k, v)).unzip()
+        // };
+        let (keys, values) = pairs.into_iter().map(|(k, v)| (k.map(|x| *x), v)).unzip();
         ast::Expr {
             location,
             end_location: Some(end_location),
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index fb181559..ced5d9b8 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -309,4 +309,10 @@ with (0 as a, 1 as b,): pass
             assert!(parse_program(source, "<test>").is_err());
         }
     }
+
+    #[test]
+    fn test_dict_containing_spread() {
+        let parse_ast = parse_expression(r#"{"k": "v", **d}"#, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
 }
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__dict_containing_spread.snap b/parser/src/snapshots/rustpython_parser__parser__tests__dict_containing_spread.snap
new file mode 100644
index 00000000..29eb0848
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__dict_containing_spread.snap
@@ -0,0 +1,81 @@
+---
+source: compiler/parser/src/parser.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 1,
+        column: 0,
+    },
+    end_location: Some(
+        Location {
+            row: 1,
+            column: 15,
+        },
+    ),
+    custom: (),
+    node: Dict {
+        keys: [
+            Some(
+                Located {
+                    location: Location {
+                        row: 1,
+                        column: 1,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 4,
+                        },
+                    ),
+                    custom: (),
+                    node: Constant {
+                        value: Str(
+                            "k",
+                        ),
+                        kind: None,
+                    },
+                },
+            ),
+            None,
+        ],
+        values: [
+            Located {
+                location: Location {
+                    row: 1,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 9,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "v",
+                    ),
+                    kind: None,
+                },
+            },
+            Located {
+                location: Location {
+                    row: 1,
+                    column: 13,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 14,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "d",
+                    ctx: Load,
+                },
+            },
+        ],
+    },
+}

From 5e7b90955274ad4c216d8d736a59d2d1460f0698 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 15 Jan 2023 13:11:55 +0900
Subject: [PATCH 389/459] Fix unparse

---
 ast/src/unparse.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ast/src/unparse.rs b/ast/src/unparse.rs
index 551283c0..b8b34209 100644
--- a/ast/src/unparse.rs
+++ b/ast/src/unparse.rs
@@ -152,7 +152,11 @@ impl<'a> Unparser<'a> {
                 let (packed, unpacked) = values.split_at(keys.len());
                 for (k, v) in keys.iter().zip(packed) {
                     self.p_delim(&mut first, ", ")?;
-                    write!(self, "{}: {}", *k, *v)?;
+                    if let Some(k) = k {
+                        write!(self, "{}: {}", *k, *v)?;
+                    } else {
+                        write!(self, "**{}", *v)?;
+                    }
                 }
                 for d in unpacked {
                     self.p_delim(&mut first, ", ")?;

From 375d592562325789f180b5e5b83d1b78aec95f47 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 15 Jan 2023 13:22:05 +0900
Subject: [PATCH 390/459] Fix scan_expression and compile_dict

---
 codegen/src/compile.rs     | 10 ++++++++--
 codegen/src/symboltable.rs |  4 +++-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 2da3aba6..c245c62d 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -1986,12 +1986,18 @@ impl Compiler {
         Ok(())
     }
 
-    fn compile_dict(&mut self, keys: &[ast::Expr], values: &[ast::Expr]) -> CompileResult<()> {
+    fn compile_dict(
+        &mut self,
+        keys: &[Option<ast::Expr>],
+        values: &[ast::Expr],
+    ) -> CompileResult<()> {
         let mut size = 0;
 
         let (packed_values, unpacked_values) = values.split_at(keys.len());
         for (key, value) in keys.iter().zip(packed_values) {
-            self.compile_expression(key)?;
+            if let Some(key) = key {
+                self.compile_expression(key)?;
+            }
             self.compile_expression(value)?;
             size += 1;
         }
diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index a12ad5f9..c12de9f5 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -887,7 +887,9 @@ impl SymbolTableBuilder {
             Dict { keys, values } => {
                 let (packed, unpacked) = values.split_at(keys.len());
                 for (key, value) in keys.iter().zip(packed) {
-                    self.scan_expression(key, context)?;
+                    if let Some(key) = key {
+                        self.scan_expression(key, context)?;
+                    }
                     self.scan_expression(value, context)?;
                 }
                 for value in unpacked {

From cb3578dbf255c073f833e9164d9f21b5d91efe04 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 15 Jan 2023 14:54:58 +0900
Subject: [PATCH 391/459] Fix scan_expression and compile_dict

---
 codegen/src/compile.rs     | 14 +++++++-------
 codegen/src/symboltable.rs | 13 +++++++------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index c245c62d..193aad40 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -1992,18 +1992,18 @@ impl Compiler {
         values: &[ast::Expr],
     ) -> CompileResult<()> {
         let mut size = 0;
-
-        let (packed_values, unpacked_values) = values.split_at(keys.len());
-        for (key, value) in keys.iter().zip(packed_values) {
-            if let Some(key) = key {
-                self.compile_expression(key)?;
-            }
+        let (packed, unpacked): (Vec<_>, Vec<_>) = keys
+            .iter()
+            .zip(values.iter())
+            .partition(|(k, _)| k.is_some());
+        for (key, value) in packed {
+            self.compile_expression(&key.as_ref().unwrap())?;
             self.compile_expression(value)?;
             size += 1;
         }
         emit!(self, Instruction::BuildMap { size });
 
-        for value in unpacked_values {
+        for (_, value) in unpacked {
             self.compile_expression(value)?;
             emit!(self, Instruction::DictUpdate);
         }
diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index c12de9f5..86a3e35e 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -885,14 +885,15 @@ impl SymbolTableBuilder {
                 self.scan_expression(value, ExpressionContext::Load)?;
             }
             Dict { keys, values } => {
-                let (packed, unpacked) = values.split_at(keys.len());
-                for (key, value) in keys.iter().zip(packed) {
-                    if let Some(key) = key {
-                        self.scan_expression(key, context)?;
-                    }
+                let (packed, unpacked): (Vec<_>, Vec<_>) = keys
+                    .iter()
+                    .zip(values.iter())
+                    .partition(|(key, _)| key.is_some());
+                for (key, value) in packed {
+                    self.scan_expression(&key.as_ref().unwrap(), context)?;
                     self.scan_expression(value, context)?;
                 }
-                for value in unpacked {
+                for (_, value) in unpacked {
                     // dict unpacking marker
                     self.scan_expression(value, context)?;
                 }

From f8ae56c75b15baa333bc33551cab7ecedde2f6a2 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 15 Jan 2023 15:04:15 +0900
Subject: [PATCH 392/459] Fix clippy errors

---
 codegen/src/compile.rs     | 2 +-
 codegen/src/symboltable.rs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 193aad40..c64cfb89 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -1997,7 +1997,7 @@ impl Compiler {
             .zip(values.iter())
             .partition(|(k, _)| k.is_some());
         for (key, value) in packed {
-            self.compile_expression(&key.as_ref().unwrap())?;
+            self.compile_expression(key.as_ref().unwrap())?;
             self.compile_expression(value)?;
             size += 1;
         }
diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 86a3e35e..125c1ea3 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -890,7 +890,7 @@ impl SymbolTableBuilder {
                     .zip(values.iter())
                     .partition(|(key, _)| key.is_some());
                 for (key, value) in packed {
-                    self.scan_expression(&key.as_ref().unwrap(), context)?;
+                    self.scan_expression(key.as_ref().unwrap(), context)?;
                     self.scan_expression(value, context)?;
                 }
                 for (_, value) in unpacked {

From 2f4fe8b3807d9cd244dd783aa81965550fee4a84 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 15 Jan 2023 16:10:51 +0900
Subject: [PATCH 393/459] Remove commented-out code

---
 parser/python.lalrpop | 43 +++++--------------------------------------
 1 file changed, 5 insertions(+), 38 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 70079057..df5057c5 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -1136,44 +1136,11 @@ Atom<Goal>: ast::Expr = {
         }.into())
     },
     <location:@L> "{" <e:DictLiteralValues?> "}" <end_location:@R> => {
-        let pairs = e.unwrap_or_default();
-
-        // let (keys, values) = match pairs.iter().position(|(k,_)| k.is_none()) {
-        //     Some(unpack_idx) => {
-        //         let mut pairs = pairs;
-        //         let (keys, mut values): (_, Vec<_>) = pairs.drain(..unpack_idx).map(|(k, v)| (*k.unwrap(), v)).unzip();
-        //
-        //         fn build_map(items: &mut Vec<(ast::Expr, ast::Expr)>) -> ast::Expr {
-        //             let location = items[0].0.location;
-        //             let end_location = items[0].0.end_location;
-        //             let (keys, values) = items.drain(..).unzip();
-        //             ast::Expr {
-        //                 location,
-        //                 end_location,
-        //                 custom: (),
-        //                 node: ast::ExprKind::Dict { keys, values }
-        //             }
-        //         }
-        //
-        //         let mut items = Vec::new();
-        //         for (key, value) in pairs.into_iter() {
-        //             if let Some(key) = key {
-        //                 items.push((*key, value));
-        //                 continue;
-        //             }
-        //             if !items.is_empty() {
-        //                 values.push(build_map(&mut items));
-        //             }
-        //             values.push(value);
-        //         }
-        //         if !items.is_empty() {
-        //             values.push(build_map(&mut items));
-        //         }
-        //         (keys, values)
-        //     },
-        //     None => pairs.into_iter().map(|(k, v)| (k, v)).unzip()
-        // };
-        let (keys, values) = pairs.into_iter().map(|(k, v)| (k.map(|x| *x), v)).unzip();
+        let (keys, values) = e
+            .unwrap_or_default()
+            .into_iter()
+            .map(|(k, v)| (k.map(|x| *x), v))
+            .unzip();
         ast::Expr {
             location,
             end_location: Some(end_location),

From d7f5dadf47ed4bfefd4b04b4fc9208ca99f052f4 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 15 Jan 2023 16:43:13 +0900
Subject: [PATCH 394/459] Add Option to Dict.keys field

---
 ast/asdl_rs.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py
index ed5800f5..a18ec651 100755
--- a/ast/asdl_rs.py
+++ b/ast/asdl_rs.py
@@ -227,12 +227,12 @@ def visitConstructor(self, cons, parent, depth):
         if cons.fields:
             self.emit(f"{cons.name} {{", depth)
             for f in cons.fields:
-                self.visit(f, parent, "", depth + 1)
+                self.visit(f, parent, "", depth + 1, cons.name)
             self.emit("},", depth)
         else:
             self.emit(f"{cons.name},", depth)
 
-    def visitField(self, field, parent, vis, depth):
+    def visitField(self, field, parent, vis, depth, constructor=None):
         typ = get_rust_type(field.type)
         fieldtype = self.typeinfo.get(field.type)
         if fieldtype and fieldtype.has_userdata:
@@ -240,7 +240,11 @@ def visitField(self, field, parent, vis, depth):
         # don't box if we're doing Vec<T>, but do box if we're doing Vec<Option<Box<T>>>
         if fieldtype and fieldtype.boxed and (not (parent.product or field.seq) or field.opt):
             typ = f"Box<{typ}>"
-        if field.opt:
+        if field.opt or (
+            # Add `Option` to allow `Dict.keys` to contain `None` for dictionary unpacking
+            # in a dict literal.
+            constructor == "Dict" and field.name == "keys"
+        ):
             typ = f"Option<{typ}>"
         if field.seq:
             typ = f"Vec<{typ}>"

From a7f1904564e97fbb54f1b2c9902c5745eafbef54 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 15 Jan 2023 16:53:13 +0900
Subject: [PATCH 395/459] Improve test

---
 parser/src/parser.rs                          |  2 +-
 ...parser__tests__dict_containing_spread.snap | 48 +++++++++++++++++--
 2 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index ced5d9b8..41945f28 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -312,7 +312,7 @@ with (0 as a, 1 as b,): pass
 
     #[test]
     fn test_dict_containing_spread() {
-        let parse_ast = parse_expression(r#"{"k": "v", **d}"#, "<test>").unwrap();
+        let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 }
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__dict_containing_spread.snap b/parser/src/snapshots/rustpython_parser__parser__tests__dict_containing_spread.snap
index 29eb0848..212f737e 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__dict_containing_spread.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__dict_containing_spread.snap
@@ -10,7 +10,7 @@ Located {
     end_location: Some(
         Location {
             row: 1,
-            column: 15,
+            column: 25,
         },
     ),
     custom: (),
@@ -31,13 +31,34 @@ Located {
                     custom: (),
                     node: Constant {
                         value: Str(
-                            "k",
+                            "a",
                         ),
                         kind: None,
                     },
                 },
             ),
             None,
+            Some(
+                Located {
+                    location: Location {
+                        row: 1,
+                        column: 16,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 19,
+                        },
+                    ),
+                    custom: (),
+                    node: Constant {
+                        value: Str(
+                            "d",
+                        ),
+                        kind: None,
+                    },
+                },
+            ),
         ],
         values: [
             Located {
@@ -54,7 +75,7 @@ Located {
                 custom: (),
                 node: Constant {
                     value: Str(
-                        "v",
+                        "b",
                     ),
                     kind: None,
                 },
@@ -72,10 +93,29 @@ Located {
                 ),
                 custom: (),
                 node: Name {
-                    id: "d",
+                    id: "c",
                     ctx: Load,
                 },
             },
+            Located {
+                location: Location {
+                    row: 1,
+                    column: 21,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 24,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "e",
+                    ),
+                    kind: None,
+                },
+            },
         ],
     },
 }

From 64736d05419a7b306d99d5f35b6458c7ab656743 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 15 Jan 2023 22:08:19 +0900
Subject: [PATCH 396/459] Simplify scan_expression

---
 codegen/src/symboltable.rs | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 125c1ea3..dd2d2954 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -885,16 +885,10 @@ impl SymbolTableBuilder {
                 self.scan_expression(value, ExpressionContext::Load)?;
             }
             Dict { keys, values } => {
-                let (packed, unpacked): (Vec<_>, Vec<_>) = keys
-                    .iter()
-                    .zip(values.iter())
-                    .partition(|(key, _)| key.is_some());
-                for (key, value) in packed {
-                    self.scan_expression(key.as_ref().unwrap(), context)?;
-                    self.scan_expression(value, context)?;
-                }
-                for (_, value) in unpacked {
-                    // dict unpacking marker
+                for (key, value) in keys.iter().zip(values.iter()) {
+                    if let Some(key) = key {
+                        self.scan_expression(key, context)?;
+                    }
                     self.scan_expression(value, context)?;
                 }
             }

From 6bda1a9e9e275a12c280e9cd35d0e74a041a671f Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 15 Jan 2023 23:20:00 +0900
Subject: [PATCH 397/459] Simplify compile_dict

---
 codegen/src/compile.rs | 26 ++++++++++----------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index c64cfb89..d5ae26d9 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -1991,23 +1991,17 @@ impl Compiler {
         keys: &[Option<ast::Expr>],
         values: &[ast::Expr],
     ) -> CompileResult<()> {
-        let mut size = 0;
-        let (packed, unpacked): (Vec<_>, Vec<_>) = keys
-            .iter()
-            .zip(values.iter())
-            .partition(|(k, _)| k.is_some());
-        for (key, value) in packed {
-            self.compile_expression(key.as_ref().unwrap())?;
-            self.compile_expression(value)?;
-            size += 1;
-        }
-        emit!(self, Instruction::BuildMap { size });
-
-        for (_, value) in unpacked {
-            self.compile_expression(value)?;
-            emit!(self, Instruction::DictUpdate);
+        emit!(self, Instruction::BuildMap { size: 0 });
+        for (key, value) in keys.iter().zip(values.iter()) {
+            if let Some(key) = key {
+                self.compile_expression(key)?;
+                self.compile_expression(value)?;
+                emit!(self, Instruction::MapAdd { i: 0 });
+            } else {
+                self.compile_expression(value)?;
+                emit!(self, Instruction::DictUpdate);
+            }
         }
-
         Ok(())
     }
 

From 4f32bacf9c4408efd7c68d47350e6787a6c4a923 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 15 Jan 2023 23:36:07 +0900
Subject: [PATCH 398/459] Rename test

---
 parser/src/parser.rs                                            | 2 +-
 ...ap => rustpython_parser__parser__tests__dict_unpacking.snap} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename parser/src/snapshots/{rustpython_parser__parser__tests__dict_containing_spread.snap => rustpython_parser__parser__tests__dict_unpacking.snap} (100%)

diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 41945f28..9761e9e1 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -311,7 +311,7 @@ with (0 as a, 1 as b,): pass
     }
 
     #[test]
-    fn test_dict_containing_spread() {
+    fn test_dict_unpacking() {
         let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__dict_containing_spread.snap b/parser/src/snapshots/rustpython_parser__parser__tests__dict_unpacking.snap
similarity index 100%
rename from parser/src/snapshots/rustpython_parser__parser__tests__dict_containing_spread.snap
rename to parser/src/snapshots/rustpython_parser__parser__tests__dict_unpacking.snap

From d83e3ee8cd4744c4a2fc3cacfb1d328621389392 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Mon, 16 Jan 2023 01:27:54 +0900
Subject: [PATCH 399/459] Revert "Simplify compile_dict"

This reverts commit b31b08a30eb24e246b7f0a0d69b2b5e56a7e7a40.
---
 codegen/src/compile.rs | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index d5ae26d9..c64cfb89 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -1991,17 +1991,23 @@ impl Compiler {
         keys: &[Option<ast::Expr>],
         values: &[ast::Expr],
     ) -> CompileResult<()> {
-        emit!(self, Instruction::BuildMap { size: 0 });
-        for (key, value) in keys.iter().zip(values.iter()) {
-            if let Some(key) = key {
-                self.compile_expression(key)?;
-                self.compile_expression(value)?;
-                emit!(self, Instruction::MapAdd { i: 0 });
-            } else {
-                self.compile_expression(value)?;
-                emit!(self, Instruction::DictUpdate);
-            }
+        let mut size = 0;
+        let (packed, unpacked): (Vec<_>, Vec<_>) = keys
+            .iter()
+            .zip(values.iter())
+            .partition(|(k, _)| k.is_some());
+        for (key, value) in packed {
+            self.compile_expression(key.as_ref().unwrap())?;
+            self.compile_expression(value)?;
+            size += 1;
         }
+        emit!(self, Instruction::BuildMap { size });
+
+        for (_, value) in unpacked {
+            self.compile_expression(value)?;
+            emit!(self, Instruction::DictUpdate);
+        }
+
         Ok(())
     }
 

From f260785da478adb7eebb7af6dd908d686a169d38 Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Mon, 16 Jan 2023 21:27:57 +0900
Subject: [PATCH 400/459] Remove useless String::from

Signed-off-by: harupy <hkawamura0130@gmail.com>
---
 parser/src/context.rs | 72 +++++++++++++++++++++----------------------
 parser/src/parser.rs  | 60 ++++++++++++++++++------------------
 2 files changed, 66 insertions(+), 66 deletions(-)

diff --git a/parser/src/context.rs b/parser/src/context.rs
index 56fdc352..f10e1054 100644
--- a/parser/src/context.rs
+++ b/parser/src/context.rs
@@ -51,127 +51,127 @@ mod tests {
 
     #[test]
     fn test_assign_name() {
-        let source = String::from("x = (1, 2, 3)");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "x = (1, 2, 3)";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_tuple() {
-        let source = String::from("(x, y) = (1, 2, 3)");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "(x, y) = (1, 2, 3)";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_list() {
-        let source = String::from("[x, y] = (1, 2, 3)");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "[x, y] = (1, 2, 3)";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_attribute() {
-        let source = String::from("x.y = (1, 2, 3)");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "x.y = (1, 2, 3)";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_subscript() {
-        let source = String::from("x[y] = (1, 2, 3)");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "x[y] = (1, 2, 3)";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_starred() {
-        let source = String::from("(x, *y) = (1, 2, 3)");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "(x, *y) = (1, 2, 3)";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_for() {
-        let source = String::from("for x in (1, 2, 3): pass");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "for x in (1, 2, 3): pass";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_list_comp() {
-        let source = String::from("x = [y for y in (1, 2, 3)]");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "x = [y for y in (1, 2, 3)]";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_set_comp() {
-        let source = String::from("x = {y for y in (1, 2, 3)}");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "x = {y for y in (1, 2, 3)}";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_with() {
-        let source = String::from("with 1 as x: pass");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "with 1 as x: pass";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_assign_named_expr() {
-        let source = String::from("if x:= 1: pass");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "if x:= 1: pass";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_ann_assign_name() {
-        let source = String::from("x: int = 1");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "x: int = 1";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_aug_assign_name() {
-        let source = String::from("x += 1");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "x += 1";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_aug_assign_attribute() {
-        let source = String::from("x.y += (1, 2, 3)");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "x.y += (1, 2, 3)";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_aug_assign_subscript() {
-        let source = String::from("x[y] += (1, 2, 3)");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "x[y] += (1, 2, 3)";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_del_name() {
-        let source = String::from("del x");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "del x";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_del_attribute() {
-        let source = String::from("del x.y");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "del x.y";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_del_subscript() {
-        let source = String::from("del x[y]");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "del x[y]";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 }
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index fb181559..82d8ded5 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -124,43 +124,43 @@ mod tests {
 
     #[test]
     fn test_parse_string() {
-        let source = String::from("'Hello world'");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "'Hello world'";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string() {
-        let source = String::from("f'Hello world'");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "f'Hello world'";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_print_hello() {
-        let source = String::from("print('Hello world')");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "print('Hello world')";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_print_2() {
-        let source = String::from("print('Hello world', 2)");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "print('Hello world', 2)";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_kwargs() {
-        let source = String::from("my_func('positional', keyword=2)");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "my_func('positional', keyword=2)";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_if_elif_else() {
-        let source = String::from("if 1: 10\nelif 2: 20\nelse: 30");
-        let parse_ast = parse_program(&source, "<test>").unwrap();
+        let source = "if 1: 10\nelif 2: 20\nelse: 30";
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
@@ -192,64 +192,64 @@ class Foo(A, B):
 
     #[test]
     fn test_parse_dict_comprehension() {
-        let source = String::from("{x1: x2 for y in z}");
-        let parse_ast = parse_expression(&source, "<test>").unwrap();
+        let source = "{x1: x2 for y in z}";
+        let parse_ast = parse_expression(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_list_comprehension() {
-        let source = String::from("[x for y in z]");
-        let parse_ast = parse_expression(&source, "<test>").unwrap();
+        let source = "[x for y in z]";
+        let parse_ast = parse_expression(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_double_list_comprehension() {
-        let source = String::from("[x for y, y2 in z for a in b if a < 5 if a > 10]");
-        let parse_ast = parse_expression(&source, "<test>").unwrap();
+        let source = "[x for y, y2 in z for a in b if a < 5 if a > 10]";
+        let parse_ast = parse_expression(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_generator_comprehension() {
-        let source = String::from("(x for y in z)");
-        let parse_ast = parse_expression(&source, "<test>").unwrap();
+        let source = "(x for y in z)";
+        let parse_ast = parse_expression(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_named_expression_generator_comprehension() {
-        let source = String::from("(x := y + 1 for y in z)");
-        let parse_ast = parse_expression(&source, "<test>").unwrap();
+        let source = "(x := y + 1 for y in z)";
+        let parse_ast = parse_expression(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_if_else_generator_comprehension() {
-        let source = String::from("(x if y else y for y in z)");
-        let parse_ast = parse_expression(&source, "<test>").unwrap();
+        let source = "(x if y else y for y in z)";
+        let parse_ast = parse_expression(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_boolop_or() {
-        let source = String::from("x or y");
-        let parse_ast = parse_expression(&source, "<test>").unwrap();
+        let source = "x or y";
+        let parse_ast = parse_expression(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_boolop_and() {
-        let source = String::from("x and y");
-        let parse_ast = parse_expression(&source, "<test>").unwrap();
+        let source = "x and y";
+        let parse_ast = parse_expression(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_slice() {
-        let source = String::from("x[1:2:3]");
-        let parse_ast = parse_expression(&source, "<test>").unwrap();
+        let source = "x[1:2:3]";
+        let parse_ast = parse_expression(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 

From b93b23b9223b4777aebe36cd9f5309265dd6adda Mon Sep 17 00:00:00 2001
From: Anders Kaseorg <andersk@mit.edu>
Date: Tue, 17 Jan 2023 22:14:24 -0500
Subject: [PATCH 401/459] Fix end location for elif blocks

Since we parse an `elif:` block as an `If` node, its location should
include its `orelse` node like it would for an `if:` block.

Signed-off-by: Anders Kaseorg <andersk@mit.edu>
---
 parser/python.lalrpop                                         | 2 +-
 .../rustpython_parser__parser__tests__parse_if_elif_else.snap | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 90f4ce25..604a4f96 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -372,7 +372,7 @@ IfStatement: ast::Stmt = {
             let x = ast::Stmt {
                 custom: (),
                 location: i.0,
-                end_location: i.4.last().unwrap().end_location,
+                end_location,
                 node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last },
             };
             last = vec![x];
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
index 7f493ddd..26bd492b 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap
@@ -79,8 +79,8 @@ expression: parse_ast
                     },
                     end_location: Some(
                         Location {
-                            row: 2,
-                            column: 10,
+                            row: 3,
+                            column: 8,
                         },
                     ),
                     custom: (),

From c157dff45447d90b2bc1fa902dda3ce8f6a0b28e Mon Sep 17 00:00:00 2001
From: harupy <hkawamura0130@gmail.com>
Date: Sun, 22 Jan 2023 00:06:52 +0900
Subject: [PATCH 402/459] Fix comment

---
 ast/asdl_rs.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py
index a18ec651..b217e629 100755
--- a/ast/asdl_rs.py
+++ b/ast/asdl_rs.py
@@ -241,8 +241,9 @@ def visitField(self, field, parent, vis, depth, constructor=None):
         if fieldtype and fieldtype.boxed and (not (parent.product or field.seq) or field.opt):
             typ = f"Box<{typ}>"
         if field.opt or (
-            # Add `Option` to allow `Dict.keys` to contain `None` for dictionary unpacking
-            # in a dict literal.
+            # When a dictionary literal contains dictionary unpacking (e.g., `{**d}`),
+            # the expression to be unpacked goes in `values` with a `None` at the corresponding
+            # position in `keys`. To handle this, the type of `keys` needs to be `Option<Vec<T>>`.
             constructor == "Dict" and field.name == "keys"
         ):
             typ = f"Option<{typ}>"

From bdc0e257b18b772ab01c42d6ab3365f4dab6c7c3 Mon Sep 17 00:00:00 2001
From: Aarni Koskela <akx@iki.fi>
Date: Fri, 20 Jan 2023 13:12:58 +0200
Subject: [PATCH 403/459] Bump phf to 0.11 series

string_cache is still using phf_shared 0.10.0 though
---
 parser/Cargo.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index 472e309b..c7b8af64 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -14,7 +14,7 @@ default = ["lalrpop"]  # removing this causes potential build failure
 [build-dependencies]
 anyhow = "1.0.45"
 lalrpop = { version = "0.19.8", optional = true }
-phf_codegen = "0.10"
+phf_codegen = "0.11.1"
 tiny-keccak = { version = "2", features = ["sha3"] }
 
 [dependencies]
@@ -27,7 +27,7 @@ lalrpop-util = "0.19.8"
 log = "0.4.16"
 num-bigint = "0.4.3"
 num-traits = "0.2.14"
-phf = "0.10.1"
+phf = "0.11.1"
 rustc-hash = "1.1.0"
 thiserror = "1.0"
 unic-emoji-char = "0.9.0"

From f4831432f98f8454348d60413dd525b740db3b4e Mon Sep 17 00:00:00 2001
From: Dimitris Fasarakis Hilliard <d.f.hilliard@gmail.com>
Date: Sun, 29 Jan 2023 19:53:00 +0200
Subject: [PATCH 404/459] Match on ascii start/continuation characters before
 calling functions.

---
 parser/src/lexer.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 1c124f62..981115d8 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -534,12 +534,15 @@ where
     }
 
     fn is_identifier_start(&self, c: char) -> bool {
-        c == '_' || is_xid_start(c)
+        match c {
+            'a'..='z' | 'A'..='Z' | '_' => true,
+            _ => is_xid_start(c),
+        }
     }
 
     fn is_identifier_continuation(&self) -> bool {
         match self.window[0] {
-            Some('_' | '0'..='9') => true,
+            Some('a'..='z' | 'A'..='Z' | '_' | '0'..='9') => true,
             Some(c) => is_xid_continue(c),
             _ => false,
         }

From 2a4d44d6673e2cadf744ae6f175cc74543ea926e Mon Sep 17 00:00:00 2001
From: Dimitris Fasarakis Hilliard <d.f.hilliard@gmail.com>
Date: Sun, 29 Jan 2023 20:08:21 +0200
Subject: [PATCH 405/459] Don't call is_emoji_presentation for each invocation
 of consume_normal

---
 parser/src/lexer.rs | 48 ++++++++++++++++++++++-----------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 981115d8..3cb28207 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -689,24 +689,12 @@ where
     fn consume_normal(&mut self) -> Result<(), LexicalError> {
         // Check if we have some character:
         if let Some(c) = self.window[0] {
-            // First check identifier:
-            if self.is_identifier_start(c) {
-                let identifier = self.lex_identifier()?;
-                self.emit(identifier);
-            } else if is_emoji_presentation(c) {
-                let tok_start = self.get_pos();
-                self.next_char();
-                let tok_end = self.get_pos();
-                self.emit((
-                    tok_start,
-                    Tok::Name {
-                        name: c.to_string(),
-                    },
-                    tok_end,
-                ));
-            } else {
-                self.consume_character(c)?;
-            }
+               if self.is_identifier_start(c) {
+                    let identifier = self.lex_identifier()?;
+                    self.emit(identifier);
+               } else {
+                    self.consume_character(c)?;
+               }
         } else {
             // We reached end of file.
             let tok_pos = self.get_pos();
@@ -1112,13 +1100,25 @@ where
                     });
                 }
             }
-
             _ => {
-                let c = self.next_char();
-                return Err(LexicalError {
-                    error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() },
-                    location: self.get_pos(),
-                });
+                 if is_emoji_presentation(c) {
+                    let tok_start = self.get_pos();
+                    self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((
+                        tok_start,
+                        Tok::Name {
+                            name: c.to_string(),
+                        },
+                        tok_end,
+                    ));
+                } else {
+                    let c = self.next_char();
+                    return Err(LexicalError {
+                        error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() },
+                        location: self.get_pos(),
+                    });
+                }
             } // Ignore all the rest..
         }
 

From d4de01c5008cde13e02a03a98b0eadd91b8e482f Mon Sep 17 00:00:00 2001
From: Dimitris Fasarakis Hilliard <d.f.hilliard@gmail.com>
Date: Sun, 29 Jan 2023 20:20:07 +0200
Subject: [PATCH 406/459] Add initial capacities, use u32s for indents/spaces.

---
 parser/src/lexer.rs | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 3cb28207..653ef032 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -18,8 +18,8 @@ use unic_ucd_ident::{is_xid_continue, is_xid_start};
 
 #[derive(Clone, Copy, PartialEq, Debug, Default)]
 struct IndentationLevel {
-    tabs: usize,
-    spaces: usize,
+    tabs: u32,
+    spaces: u32,
 }
 
 impl IndentationLevel {
@@ -225,7 +225,8 @@ where
             at_begin_of_line: true,
             nesting: 0,
             indentations: Indentations::default(),
-            pending: Vec::new(),
+            // Usually we have less than 5 tokens pending.
+            pending: Vec::with_capacity(5),
             location: start,
             window: CharWindow::new(input),
         };
@@ -257,13 +258,13 @@ where
         };
 
         let start_pos = self.get_pos();
-        let mut name = String::new();
+        let mut name = String::with_capacity(8);
         while self.is_identifier_continuation() {
             name.push(self.next_char().unwrap());
         }
         let end_pos = self.get_pos();
 
-        if let Some(tok) = KEYWORDS.get(name.as_str()) {
+        if let Some(tok) = KEYWORDS.get(&name) {
             Ok((start_pos, tok.clone(), end_pos))
         } else {
             Ok((start_pos, Tok::Name { name }, end_pos))
@@ -464,7 +465,7 @@ where
             self.next_char();
         }
         let quote_char = self.next_char().unwrap();
-        let mut string_content = String::new();
+        let mut string_content = String::with_capacity(5);
 
         // If the next two characters are also the quote character, then we have a triple-quoted
         // string; consume those two characters and ensure that we require a triple-quote to close
@@ -567,8 +568,8 @@ where
     /// Given we are at the start of a line, count the number of spaces and/or tabs until the first character.
     fn eat_indentation(&mut self) -> Result<IndentationLevel, LexicalError> {
         // Determine indentation:
-        let mut spaces: usize = 0;
-        let mut tabs: usize = 0;
+        let mut spaces: u32 = 0;
+        let mut tabs: u32 = 0;
         loop {
             match self.window[0] {
                 Some(' ') => {
@@ -1127,7 +1128,7 @@ where
 
     fn eat_single_char(&mut self, ty: Tok) {
         let tok_start = self.get_pos();
-        self.next_char().unwrap();
+        self.next_char();
         let tok_end = self.get_pos();
         self.emit((tok_start, ty, tok_end));
     }

From c8005694c7aeb0d3c59e87a9ab17cc1d346dd4b8 Mon Sep 17 00:00:00 2001
From: Dimitris Fasarakis Hilliard <d.f.hilliard@gmail.com>
Date: Mon, 30 Jan 2023 21:42:59 +0200
Subject: [PATCH 407/459] Eat for comma.

---
 parser/src/lexer.rs | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 653ef032..61bff370 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -690,12 +690,12 @@ where
     fn consume_normal(&mut self) -> Result<(), LexicalError> {
         // Check if we have some character:
         if let Some(c) = self.window[0] {
-               if self.is_identifier_start(c) {
-                    let identifier = self.lex_identifier()?;
-                    self.emit(identifier);
-               } else {
-                    self.consume_character(c)?;
-               }
+            if self.is_identifier_start(c) {
+                let identifier = self.lex_identifier()?;
+                self.emit(identifier);
+            } else {
+                self.consume_character(c)?;
+            }
         } else {
             // We reached end of file.
             let tok_pos = self.get_pos();
@@ -1039,10 +1039,7 @@ where
                 }
             }
             ',' => {
-                let tok_start = self.get_pos();
-                self.next_char();
-                let tok_end = self.get_pos();
-                self.emit((tok_start, Tok::Comma, tok_end));
+                self.eat_single_char(Tok::Comma);
             }
             '.' => {
                 if let Some('0'..='9') = self.window[1] {
@@ -1102,7 +1099,7 @@ where
                 }
             }
             _ => {
-                 if is_emoji_presentation(c) {
+                if is_emoji_presentation(c) {
                     let tok_start = self.get_pos();
                     self.next_char();
                     let tok_end = self.get_pos();

From bff9884df185fb0e3c03a981d37efad177dab5f3 Mon Sep 17 00:00:00 2001
From: Dimitris Fasarakis Hilliard <d.f.hilliard@gmail.com>
Date: Wed, 1 Feb 2023 12:17:33 +0200
Subject: [PATCH 408/459] Hint that the unwrap should always succeed.

---
 parser/src/lexer.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 61bff370..5eeaf885 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -1125,7 +1125,11 @@ where
 
     fn eat_single_char(&mut self, ty: Tok) {
         let tok_start = self.get_pos();
-        self.next_char();
+        self.next_char().unwrap_or_else(|| unsafe {
+            // SAFETY: eat_single_char has been called only after a character has been read
+            // from the window, so the window is guaranteed to be non-empty.
+            std::hint::unreachable_unchecked()
+        });
         let tok_end = self.get_pos();
         self.emit((tok_start, ty, tok_end));
     }

From 00dc7ec7095902a47c8c877ac1b30e8cd1fa208d Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 6 Feb 2023 14:18:16 +0900
Subject: [PATCH 409/459] use workspace dependencies

---
 Cargo.toml         |  2 +-
 ast/Cargo.toml     |  3 ++-
 codegen/Cargo.toml | 18 +++++++++---------
 core/Cargo.toml    | 19 ++++++++++---------
 parser/Cargo.toml  | 23 ++++++++++++-----------
 5 files changed, 34 insertions(+), 31 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 34dbad32..b6e204df 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,4 +10,4 @@ rustpython-compiler-core = { path = "core" }
 rustpython-codegen = { path = "codegen" }
 rustpython-parser = { path = "parser" }
 
-thiserror = "1.0"
+thiserror = { workspace = true }
diff --git a/ast/Cargo.toml b/ast/Cargo.toml
index 08f29fcc..bb280f48 100644
--- a/ast/Cargo.toml
+++ b/ast/Cargo.toml
@@ -14,6 +14,7 @@ fold = []
 unparse = ["rustpython-common"]
 
 [dependencies]
-num-bigint = "0.4.3"
 rustpython-compiler-core = { path = "../core", version = "0.2.0" }
 rustpython-common = { path = "../../common", version = "0.2.0", optional = true }
+
+num-bigint = { workspace = true }
diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml
index 4265e4b2..fd6ad653 100644
--- a/codegen/Cargo.toml
+++ b/codegen/Cargo.toml
@@ -11,16 +11,16 @@ edition = "2021"
 rustpython-ast = { path = "../ast", features = ["unparse"] }
 rustpython-compiler-core = { path = "../core", version = "0.2.0" }
 
-ahash = "0.7.6"
-bitflags = "1.3.2"
-indexmap = "1.8.1"
-itertools = "0.10.3"
-log = "0.4.16"
-num-complex = { version = "0.4.0", features = ["serde"] }
-num-traits = "0.2.14"
-thiserror = "1.0"
+ahash = { workspace = true }
+bitflags = { workspace = true }
+indexmap = { workspace = true }
+itertools = { workspace = true }
+log = { workspace = true }
+num-complex = { workspace = true, features = ["serde"] }
+num-traits = { workspace = true }
+thiserror = { workspace = true }
 
 [dev-dependencies]
 rustpython-parser = { path = "../parser" }
 
-insta = "1.14.0"
+insta = { workspace = true }
diff --git a/core/Cargo.toml b/core/Cargo.toml
index f9cbce31..bf385a71 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -8,13 +8,14 @@ repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
 [dependencies]
-bincode = "1.3.3"
-bitflags = "1.3.2"
-bstr = "0.2.17"
-itertools = "0.10.3"
+bincode = { workspace = true }
+bitflags = { workspace = true }
+bstr = { workspace = true }
+itertools = { workspace = true }
+num-bigint = { workspace = true, features = ["serde"] }
+num-complex = { workspace = true, features = ["serde"] }
+num_enum = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+thiserror = { workspace = true }
+
 lz4_flex = "0.9.2"
-num-bigint = { version = "0.4.3", features = ["serde"] }
-num-complex = { version = "0.4.0", features = ["serde"] }
-num_enum = "0.5.7"
-serde = { version = "1.0.136", features = ["derive"] }
-thiserror = "1.0"
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index c7b8af64..56f3d161 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -12,7 +12,7 @@ edition = "2021"
 default = ["lalrpop"]  # removing this causes potential build failure
 
 [build-dependencies]
-anyhow = "1.0.45"
+anyhow = { workspace = true }
 lalrpop = { version = "0.19.8", optional = true }
 phf_codegen = "0.11.1"
 tiny-keccak = { version = "2", features = ["sha3"] }
@@ -21,18 +21,19 @@ tiny-keccak = { version = "2", features = ["sha3"] }
 rustpython-ast = { path = "../ast", version = "0.2.0" }
 rustpython-compiler-core = { path = "../core", version = "0.2.0" }
 
-ahash = "0.7.6"
-itertools = "0.10.3"
-lalrpop-util = "0.19.8"
-log = "0.4.16"
-num-bigint = "0.4.3"
-num-traits = "0.2.14"
-phf = "0.11.1"
-rustc-hash = "1.1.0"
-thiserror = "1.0"
+ahash = { workspace = true }
+itertools = { workspace = true }
+log = { workspace = true }
+num-bigint = { workspace = true }
+num-traits = { workspace = true }
+thiserror = { workspace = true }
+
 unic-emoji-char = "0.9.0"
 unic-ucd-ident  = "0.9.0"
 unicode_names2 = "0.5.0"
+lalrpop-util = "0.19.8"
+phf = "0.11.1"
+rustc-hash = "1.1.0"
 
 [dev-dependencies]
-insta = "1.14.0"
+insta = { workspace = true }

From 639e8fc52fb7b0edfe979165614e42af67d09925 Mon Sep 17 00:00:00 2001
From: Dimitris Fasarakis Hilliard <d.f.hilliard@gmail.com>
Date: Mon, 6 Feb 2023 06:32:02 +0200
Subject: [PATCH 410/459] Move NewLineHandler inline, don't check each
 character twice.

---
 parser/src/lexer.rs | 146 ++++++++++++++++++--------------------------
 1 file changed, 59 insertions(+), 87 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 5eeaf885..360dd9c3 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -115,10 +115,6 @@ where
         *self.window.last_mut().expect("never empty") = next;
         next
     }
-
-    fn change_first(&mut self, ch: char) {
-        *self.window.first_mut().expect("never empty") = Some(ch);
-    }
 }
 
 impl<T, const N: usize, Idx> Index<Idx> for CharWindow<T, N>
@@ -135,7 +131,6 @@ where
 
 pub struct Lexer<T: Iterator<Item = char>> {
     window: CharWindow<T, 3>,
-
     at_begin_of_line: bool,
     nesting: usize, // Amount of parenthesis
     indentations: Indentations,
@@ -160,60 +155,7 @@ pub fn make_tokenizer_located(
     source: &str,
     start_location: Location,
 ) -> impl Iterator<Item = LexResult> + '_ {
-    let nlh = NewlineHandler::new(source.chars());
-    Lexer::new(nlh, start_location)
-}
-
-// The newline handler is an iterator which collapses different newline
-// types into \n always.
-pub struct NewlineHandler<T: Iterator<Item = char>> {
-    window: CharWindow<T, 2>,
-}
-
-impl<T> NewlineHandler<T>
-where
-    T: Iterator<Item = char>,
-{
-    pub fn new(source: T) -> Self {
-        let mut nlh = NewlineHandler {
-            window: CharWindow::new(source),
-        };
-        nlh.shift();
-        nlh.shift();
-        nlh
-    }
-
-    fn shift(&mut self) -> Option<char> {
-        let result = self.window[0];
-        self.window.slide();
-        result
-    }
-}
-
-impl<T> Iterator for NewlineHandler<T>
-where
-    T: Iterator<Item = char>,
-{
-    type Item = char;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        // Collapse \r\n into \n
-        loop {
-            match self.window[..2] {
-                [Some('\r'), Some('\n')] => {
-                    // Windows EOL into \n
-                    self.shift();
-                }
-                [Some('\r'), _] => {
-                    // MAC EOL into \n
-                    self.window.change_first('\n');
-                }
-                _ => break,
-            }
-        }
-
-        self.shift()
-    }
+    Lexer::new(source.chars(), start_location)
 }
 
 impl<T> Lexer<T>
@@ -446,10 +388,9 @@ where
     fn lex_comment(&mut self) -> LexResult {
         let start_pos = self.get_pos();
         let mut value = String::new();
-        value.push(self.next_char().unwrap());
         loop {
             match self.window[0] {
-                Some('\n') | None => {
+                Some('\n' | '\r') | None => {
                     let end_pos = self.get_pos();
                     return Ok((start_pos, Tok::Comment(value), end_pos));
                 }
@@ -487,7 +428,6 @@ where
                             continue;
                         }
                     }
-
                     if c == '\n' && !triple_quoted {
                         return Err(LexicalError {
                             error: LexicalErrorType::OtherError(
@@ -613,7 +553,7 @@ where
                     spaces = 0;
                     tabs = 0;
                 }
-                Some('\n') => {
+                Some('\n' | '\r') => {
                     // Empty line!
                     self.next_char();
                     spaces = 0;
@@ -1059,7 +999,7 @@ where
                     }
                 }
             }
-            '\n' => {
+            '\n' | '\r' => {
                 let tok_start = self.get_pos();
                 self.next_char();
                 let tok_end = self.get_pos();
@@ -1082,13 +1022,16 @@ where
             }
             '\\' => {
                 self.next_char();
-                if let Some('\n') = self.window[0] {
-                    self.next_char();
-                } else {
-                    return Err(LexicalError {
-                        error: LexicalErrorType::LineContinuationError,
-                        location: self.get_pos(),
-                    });
+                match self.window[0] {
+                    Some('\n' | '\r') => {
+                        self.next_char();
+                    }
+                    _ => {
+                        return Err(LexicalError {
+                            error: LexicalErrorType::LineContinuationError,
+                            location: self.get_pos(),
+                        })
+                    }
                 }
 
                 if self.window[0].is_none() {
@@ -1136,12 +1079,22 @@ where
 
     /// Helper function to go to the next character coming up.
     fn next_char(&mut self) -> Option<char> {
-        let c = self.window[0];
+        let mut c = self.window[0];
         self.window.slide();
-        if c == Some('\n') {
-            self.location.newline();
-        } else {
-            self.location.go_right();
+        match c {
+            Some('\n') => {
+                self.location.newline();
+            }
+            Some('\r') => {
+                if self.window[0] == Some('\n') {
+                    self.window.slide();
+                }
+                self.location.newline();
+                c = Some('\n');
+            }
+            _ => {
+                self.location.go_right();
+            }
         }
         c
     }
@@ -1189,7 +1142,7 @@ where
 
 #[cfg(test)]
 mod tests {
-    use super::{make_tokenizer, NewlineHandler, StringKind, Tok};
+    use super::{make_tokenizer, StringKind, Tok};
     use num_bigint::BigInt;
 
     const WINDOWS_EOL: &str = "\r\n";
@@ -1201,16 +1154,6 @@ mod tests {
         lexer.map(|x| x.unwrap().1).collect()
     }
 
-    #[test]
-    fn test_newline_processor() {
-        // Escape \ followed by \n (by removal):
-        let src = "b\\\r\n";
-        assert_eq!(4, src.len());
-        let nlh = NewlineHandler::new(src.chars());
-        let x: Vec<char> = nlh.collect();
-        assert_eq!(vec!['b', '\\', '\n'], x);
-    }
-
     fn stok(s: &str) -> Tok {
         Tok::String {
             value: s.to_owned(),
@@ -1645,4 +1588,33 @@ mod tests {
         let tokens = lex_source(source);
         assert_eq!(tokens, vec![stok(r"\N{EN SPACE}"), Tok::Newline])
     }
+
+    macro_rules! test_triple_quoted {
+        ($($name:ident: $eol:expr,)*) => {
+        $(
+            #[test]
+            fn $name() {
+                let source = format!("\"\"\"{0} test string{0} \"\"\"", $eol);
+                let tokens = lex_source(&source);
+                assert_eq!(
+                    tokens,
+                    vec![
+                        Tok::String {
+                            value: "\n test string\n ".to_owned(),
+                            kind: StringKind::String,
+                            triple_quoted: true,
+                        },
+                        Tok::Newline,
+                    ]
+                )
+            }
+        )*
+        }
+    }
+
+    test_triple_quoted! {
+        test_triple_quoted_windows_eol: WINDOWS_EOL,
+        test_triple_quoted_mac_eol: MAC_EOL,
+        test_triple_quoted_unix_eol: UNIX_EOL,
+    }
 }

From 31e0962c66b336dc174e9e799274cb756e0f4272 Mon Sep 17 00:00:00 2001
From: Dimitris Fasarakis Hilliard <d.f.hilliard@gmail.com>
Date: Tue, 7 Feb 2023 20:59:35 +0200
Subject: [PATCH 411/459] Document lexer.

---
 parser/src/lexer.rs | 117 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 92 insertions(+), 25 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 360dd9c3..20e6b148 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -1,7 +1,37 @@
-//! This module takes care of lexing python source text.
+//! This module takes care of lexing Python source text.
 //!
-//! This means source code is translated into separate tokens.
-
+//! This means source code is scanned and translated into separate tokens. The rules
+//! governing what is and is not a valid token are defined in the Python reference
+//! guide section on [Lexical analysis].
+//!
+//! The primary function in this module is [`make_tokenizer`], which takes a string slice
+//! and returns an iterator over the tokens in the source code. The tokens are currently returned
+//! as a `Result<Spanned, LexicalError>`, where [`Spanned`] is a tuple containing the
+//! start and end [`Location`] and a [`Tok`] denoting the token.
+//!
+//! # Example
+//!
+//! ```
+//! use rustpython_parser::lexer::{make_tokenizer, Tok};
+//! use rustpython_parser::token::StringKind;
+//!
+//! let source = "x = 'RustPython'";
+//! let tokens = make_tokenizer(source)
+//!     .map(|tok| tok.expect("Failed to lex"))
+//!     .collect::<Vec<_>>();
+//!
+//! for (start, token, end) in tokens {
+//!     println!(
+//!         "{0},{1}-{2},{3:<5} {token:?}",
+//!         start.row(),
+//!         start.column(),
+//!         end.row(),
+//!         end.column(),
+//!     );
+//! }
+//! ```
+//!
+//! [Lexical analysis]: https://docs.python.org/3/reference/lexical_analysis.html
 pub use super::token::{StringKind, Tok};
 use crate::ast::Location;
 use crate::error::{LexicalError, LexicalErrorType};
@@ -16,6 +46,8 @@ use std::str::FromStr;
 use unic_emoji_char::is_emoji_presentation;
 use unic_ucd_ident::{is_xid_continue, is_xid_start};
 
+// Indentations are tracked by a stack of indentation levels. IndentationLevel keeps
+// track of the number of tabs and spaces at the current level.
 #[derive(Clone, Copy, PartialEq, Debug, Default)]
 struct IndentationLevel {
     tabs: u32,
@@ -57,6 +89,9 @@ impl IndentationLevel {
     }
 }
 
+// The indentations stack is used to keep track of the current indentation level.
+// Similar to the CPython implementation, the Indentations stack always has at
+// least one level which is never popped. See Reference 2.1.8.
 #[derive(Debug)]
 struct Indentations {
     indent_stack: Vec<IndentationLevel>,
@@ -93,6 +128,8 @@ impl Default for Indentations {
     }
 }
 
+// A CharWindow is a sliding window over an iterator of chars. It is used to
+// allow for look-ahead when scanning tokens from the source code.
 struct CharWindow<T: Iterator<Item = char>, const N: usize> {
     source: T,
     window: [Option<char>; N],
@@ -129,28 +166,53 @@ where
     }
 }
 
+/// A lexer for Python source code.
 pub struct Lexer<T: Iterator<Item = char>> {
+    // Contains the source code to be lexed.
     window: CharWindow<T, 3>,
+    // Are we at the beginning of a line?
     at_begin_of_line: bool,
-    nesting: usize, // Amount of parenthesis
+    // Amount of parenthesis.
+    nesting: usize,
+    // Indentation levels.
     indentations: Indentations,
-
+    // Pending list of tokens to be returned.
     pending: Vec<Spanned>,
+    // The current location.
     location: Location,
 }
 
 // generated in build.rs, in gen_phf()
+/// A map of keywords to their tokens.
 pub static KEYWORDS: phf::Map<&'static str, Tok> =
     include!(concat!(env!("OUT_DIR"), "/keywords.rs"));
 
+/// Contains a Token along with its start and end location.
 pub type Spanned = (Location, Tok, Location);
+/// The result of lexing a token.
 pub type LexResult = Result<Spanned, LexicalError>;
 
+/// Create a new tokenizer from a source string.
+///
+/// # Examples
+///
+/// ```
+/// use rustpython_parser::lexer::{make_tokenizer};
+///
+/// let source = "def hello(): return 'world'";
+/// let tokenizer = make_tokenizer(source);
+///
+/// for token in tokenizer {
+///    println!("{:?}", token);
+/// }
+/// ```
 #[inline]
 pub fn make_tokenizer(source: &str) -> impl Iterator<Item = LexResult> + '_ {
     make_tokenizer_located(source, Location::default())
 }
 
+/// Create a new tokenizer from a source string, starting at a given location.
+/// You probably want to use [`make_tokenizer`] instead.
 pub fn make_tokenizer_located(
     source: &str,
     start_location: Location,
@@ -162,6 +224,8 @@ impl<T> Lexer<T>
 where
     T: Iterator<Item = char>,
 {
+    /// Create a new lexer from T and a starting location. You probably want to use
+    /// [`make_tokenizer`] instead.
     pub fn new(input: T, start: Location) -> Self {
         let mut lxr = Lexer {
             at_begin_of_line: true,
@@ -172,6 +236,7 @@ where
             location: start,
             window: CharWindow::new(input),
         };
+        // Fill the window.
         lxr.window.slide();
         lxr.window.slide();
         lxr.window.slide();
@@ -182,7 +247,7 @@ where
         lxr
     }
 
-    // Lexer helper functions:
+    /// Lex an identifier. Also used for keywords and string/bytes literals with a prefix.
     fn lex_identifier(&mut self) -> LexResult {
         // Detect potential string like rb'' b'' f'' u'' r''
         match self.window[..3] {
@@ -384,7 +449,7 @@ where
         }
     }
 
-    /// Skip everything until end of line
+    /// Lex a single comment.
     fn lex_comment(&mut self) -> LexResult {
         let start_pos = self.get_pos();
         let mut value = String::new();
@@ -400,6 +465,7 @@ where
         }
     }
 
+    /// Lex a string literal.
     fn lex_string(&mut self, kind: StringKind) -> LexResult {
         let start_pos = self.get_pos();
         for _ in 0..kind.prefix_len() {
@@ -474,6 +540,8 @@ where
         Ok((start_pos, tok, end_pos))
     }
 
+    // Checks if the character c is a valid starting character as described
+    // in https://docs.python.org/3/reference/lexical_analysis.html#identifiers
     fn is_identifier_start(&self, c: char) -> bool {
         match c {
             'a'..='z' | 'A'..='Z' | '_' => true,
@@ -481,6 +549,8 @@ where
         }
     }
 
+    // Checks if the character c is a valid continuation character as described
+    // in https://docs.python.org/3/reference/lexical_analysis.html#identifiers
     fn is_identifier_continuation(&self) -> bool {
         match self.window[0] {
             Some('a'..='z' | 'A'..='Z' | '_' | '0'..='9') => true,
@@ -489,8 +559,8 @@ where
         }
     }
 
-    /// This is the main entry point. Call this function to retrieve the next token.
-    /// This function is used by the iterator implementation.
+    // This is the main entry point. Call this function to retrieve the next token.
+    // This function is used by the iterator implementation.
     fn inner_next(&mut self) -> LexResult {
         // top loop, keep on processing, until we have something pending.
         while self.pending.is_empty() {
@@ -505,7 +575,7 @@ where
         Ok(self.pending.remove(0))
     }
 
-    /// Given we are at the start of a line, count the number of spaces and/or tabs until the first character.
+    // Given we are at the start of a line, count the number of spaces and/or tabs until the first character.
     fn eat_indentation(&mut self) -> Result<IndentationLevel, LexicalError> {
         // Determine indentation:
         let mut spaces: u32 = 0;
@@ -574,6 +644,7 @@ where
         Ok(IndentationLevel { tabs, spaces })
     }
 
+    // Push/pop indents/dedents based on the current indentation level.
     fn handle_indentations(&mut self) -> Result<(), LexicalError> {
         let indentation_level = self.eat_indentation()?;
 
@@ -626,10 +697,10 @@ where
         Ok(())
     }
 
-    /// Take a look at the next character, if any, and decide upon the next steps.
+    // Take a look at the next character, if any, and decide upon the next steps.
     fn consume_normal(&mut self) -> Result<(), LexicalError> {
-        // Check if we have some character:
         if let Some(c) = self.window[0] {
+            // Identifiers are the most common case.
             if self.is_identifier_start(c) {
                 let identifier = self.lex_identifier()?;
                 self.emit(identifier);
@@ -666,7 +737,7 @@ where
         Ok(())
     }
 
-    /// Okay, we are facing a weird character, what is it? Determine that.
+    // Dispatch based on the given character.
     fn consume_character(&mut self, c: char) -> Result<(), LexicalError> {
         match c {
             '0'..='9' => {
@@ -1060,12 +1131,13 @@ where
                         location: self.get_pos(),
                     });
                 }
-            } // Ignore all the rest..
+            }
         }
 
         Ok(())
     }
 
+    // Used by single character tokens to advance the window and emit the correct token.
     fn eat_single_char(&mut self, ty: Tok) {
         let tok_start = self.get_pos();
         self.next_char().unwrap_or_else(|| unsafe {
@@ -1077,7 +1149,7 @@ where
         self.emit((tok_start, ty, tok_end));
     }
 
-    /// Helper function to go to the next character coming up.
+    // Helper function to go to the next character coming up.
     fn next_char(&mut self) -> Option<char> {
         let mut c = self.window[0];
         self.window.slide();
@@ -1099,22 +1171,20 @@ where
         c
     }
 
-    /// Helper function to retrieve the current position.
+    // Helper function to retrieve the current position.
     fn get_pos(&self) -> Location {
         self.location
     }
 
-    /// Helper function to emit a lexed token to the queue of tokens.
+    // Helper function to emit a lexed token to the queue of tokens.
     fn emit(&mut self, spanned: Spanned) {
         self.pending.push(spanned);
     }
 }
 
-/* Implement iterator pattern for the get_tok function.
-
-Calling the next element in the iterator will yield the next lexical
-token.
-*/
+// Implement iterator pattern for Lexer.
+// Calling the next element in the iterator will yield the next lexical
+// token.
 impl<T> Iterator for Lexer<T>
 where
     T: Iterator<Item = char>,
@@ -1122,9 +1192,6 @@ where
     type Item = LexResult;
 
     fn next(&mut self) -> Option<Self::Item> {
-        // Idea: create some sort of hash map for single char tokens:
-        // let mut X = HashMap::new();
-        // X.insert('=', Tok::Equal);
         let token = self.inner_next();
         trace!(
             "Lex token {:?}, nesting={:?}, indent stack: {:?}",

From dc78f6033b34d9f9dcd895858059fe3b5240ecc8 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Fri, 10 Feb 2023 10:39:40 -0500
Subject: [PATCH 412/459] Use entire range for generators-as-arguments

---
 parser/python.lalrpop | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 719cef46..78b71361 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -1274,11 +1274,11 @@ ArgumentList: ArgumentList = {
 };
 
 FunctionArgument: (Option<(ast::Location, ast::Location, Option<String>)>, ast::Expr) = {
-    <e:NamedExpressionTest> <c:CompFor?> => {
+    <location:@L> <e:NamedExpressionTest> <c:CompFor?> <end_location:@R> => {
         let expr = match c {
             Some(c) => ast::Expr {
-                location: e.location,
-                end_location: e.end_location,
+                location,
+                end_location: Some(end_location),
                 custom: (),
                 node: ast::ExprKind::GeneratorExp {
                     elt: Box::new(e),

From 4ee4fcf40718aab36e471a098b37ee8b77cadcff Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Sat, 11 Feb 2023 05:20:06 +0900
Subject: [PATCH 413/459] Add test_generator_expression_argument

---
 parser/src/parser.rs                          |  13 +
 ..._tests__generator_expression_argument.snap | 333 ++++++++++++++++++
 2 files changed, 346 insertions(+)
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__generator_expression_argument.snap

diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 8abbcacf..38b065a5 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -310,6 +310,19 @@ with (0 as a, 1 as b,): pass
         }
     }
 
+    #[test]
+    fn test_generator_expression_argument() {
+        let source = r#"' '.join(
+    sql
+    for sql in (
+        "LIMIT %d" % limit if limit else None,
+        ("OFFSET %d" % offset) if offset else None,
+    )
+)"#;
+        let parse_ast = parse_expression(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
     #[test]
     fn test_dict_unpacking() {
         let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "<test>").unwrap();
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__generator_expression_argument.snap b/parser/src/snapshots/rustpython_parser__parser__tests__generator_expression_argument.snap
new file mode 100644
index 00000000..863ece46
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__generator_expression_argument.snap
@@ -0,0 +1,333 @@
+---
+source: compiler/parser/src/parser.rs
+expression: parse_ast
+---
+Located {
+    location: Location {
+        row: 1,
+        column: 0,
+    },
+    end_location: Some(
+        Location {
+            row: 7,
+            column: 1,
+        },
+    ),
+    custom: (),
+    node: Call {
+        func: Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 8,
+                },
+            ),
+            custom: (),
+            node: Attribute {
+                value: Located {
+                    location: Location {
+                        row: 1,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 3,
+                        },
+                    ),
+                    custom: (),
+                    node: Constant {
+                        value: Str(
+                            " ",
+                        ),
+                        kind: None,
+                    },
+                },
+                attr: "join",
+                ctx: Load,
+            },
+        },
+        args: [
+            Located {
+                location: Location {
+                    row: 2,
+                    column: 4,
+                },
+                end_location: Some(
+                    Location {
+                        row: 6,
+                        column: 5,
+                    },
+                ),
+                custom: (),
+                node: GeneratorExp {
+                    elt: Located {
+                        location: Location {
+                            row: 2,
+                            column: 4,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 2,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Name {
+                            id: "sql",
+                            ctx: Load,
+                        },
+                    },
+                    generators: [
+                        Comprehension {
+                            target: Located {
+                                location: Location {
+                                    row: 3,
+                                    column: 8,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 3,
+                                        column: 11,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "sql",
+                                    ctx: Store,
+                                },
+                            },
+                            iter: Located {
+                                location: Location {
+                                    row: 3,
+                                    column: 15,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 6,
+                                        column: 5,
+                                    },
+                                ),
+                                custom: (),
+                                node: Tuple {
+                                    elts: [
+                                        Located {
+                                            location: Location {
+                                                row: 4,
+                                                column: 8,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 4,
+                                                    column: 45,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: IfExp {
+                                                test: Located {
+                                                    location: Location {
+                                                        row: 4,
+                                                        column: 30,
+                                                    },
+                                                    end_location: Some(
+                                                        Location {
+                                                            row: 4,
+                                                            column: 35,
+                                                        },
+                                                    ),
+                                                    custom: (),
+                                                    node: Name {
+                                                        id: "limit",
+                                                        ctx: Load,
+                                                    },
+                                                },
+                                                body: Located {
+                                                    location: Location {
+                                                        row: 4,
+                                                        column: 8,
+                                                    },
+                                                    end_location: Some(
+                                                        Location {
+                                                            row: 4,
+                                                            column: 26,
+                                                        },
+                                                    ),
+                                                    custom: (),
+                                                    node: BinOp {
+                                                        left: Located {
+                                                            location: Location {
+                                                                row: 4,
+                                                                column: 8,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 4,
+                                                                    column: 18,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: Constant {
+                                                                value: Str(
+                                                                    "LIMIT %d",
+                                                                ),
+                                                                kind: None,
+                                                            },
+                                                        },
+                                                        op: Mod,
+                                                        right: Located {
+                                                            location: Location {
+                                                                row: 4,
+                                                                column: 21,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 4,
+                                                                    column: 26,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: Name {
+                                                                id: "limit",
+                                                                ctx: Load,
+                                                            },
+                                                        },
+                                                    },
+                                                },
+                                                orelse: Located {
+                                                    location: Location {
+                                                        row: 4,
+                                                        column: 41,
+                                                    },
+                                                    end_location: Some(
+                                                        Location {
+                                                            row: 4,
+                                                            column: 45,
+                                                        },
+                                                    ),
+                                                    custom: (),
+                                                    node: Constant {
+                                                        value: None,
+                                                        kind: None,
+                                                    },
+                                                },
+                                            },
+                                        },
+                                        Located {
+                                            location: Location {
+                                                row: 5,
+                                                column: 8,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 5,
+                                                    column: 50,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: IfExp {
+                                                test: Located {
+                                                    location: Location {
+                                                        row: 5,
+                                                        column: 34,
+                                                    },
+                                                    end_location: Some(
+                                                        Location {
+                                                            row: 5,
+                                                            column: 40,
+                                                        },
+                                                    ),
+                                                    custom: (),
+                                                    node: Name {
+                                                        id: "offset",
+                                                        ctx: Load,
+                                                    },
+                                                },
+                                                body: Located {
+                                                    location: Location {
+                                                        row: 5,
+                                                        column: 9,
+                                                    },
+                                                    end_location: Some(
+                                                        Location {
+                                                            row: 5,
+                                                            column: 29,
+                                                        },
+                                                    ),
+                                                    custom: (),
+                                                    node: BinOp {
+                                                        left: Located {
+                                                            location: Location {
+                                                                row: 5,
+                                                                column: 9,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 5,
+                                                                    column: 20,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: Constant {
+                                                                value: Str(
+                                                                    "OFFSET %d",
+                                                                ),
+                                                                kind: None,
+                                                            },
+                                                        },
+                                                        op: Mod,
+                                                        right: Located {
+                                                            location: Location {
+                                                                row: 5,
+                                                                column: 23,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 5,
+                                                                    column: 29,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: Name {
+                                                                id: "offset",
+                                                                ctx: Load,
+                                                            },
+                                                        },
+                                                    },
+                                                },
+                                                orelse: Located {
+                                                    location: Location {
+                                                        row: 5,
+                                                        column: 46,
+                                                    },
+                                                    end_location: Some(
+                                                        Location {
+                                                            row: 5,
+                                                            column: 50,
+                                                        },
+                                                    ),
+                                                    custom: (),
+                                                    node: Constant {
+                                                        value: None,
+                                                        kind: None,
+                                                    },
+                                                },
+                                            },
+                                        },
+                                    ],
+                                    ctx: Load,
+                                },
+                            },
+                            ifs: [],
+                            is_async: 0,
+                        },
+                    ],
+                },
+            },
+        ],
+        keywords: [],
+    },
+}

From e6f361d2a532b91ae034daed999e0b5f9654c53e Mon Sep 17 00:00:00 2001
From: Dimitris Fasarakis Hilliard <d.f.hilliard@gmail.com>
Date: Sat, 11 Feb 2023 17:56:54 +0200
Subject: [PATCH 414/459] Refactor: Join string and string_parser.

---
 parser/src/lib.rs                             |   3 +-
 ...__fstring_parse_selfdocumenting_base.snap} |   3 +-
 ...ring_parse_selfdocumenting_base_more.snap} |   3 +-
 ...fstring_parse_selfdocumenting_format.snap} |   3 +-
 ...r__string__tests__parse_empty_fstring.snap |   6 +
 ...parser__string__tests__parse_fstring.snap} |   3 +-
 ..._string__tests__parse_fstring_equals.snap} |   3 +-
 ...ng__tests__parse_fstring_nested_spec.snap} |   3 +-
 ...ing__tests__parse_fstring_not_equals.snap} |   3 +-
 ...tests__parse_fstring_not_nested_spec.snap} |   3 +-
 ...ts__parse_fstring_selfdoc_prec_space.snap} |   3 +-
 ...parse_fstring_selfdoc_trailing_space.snap} |   3 +-
 ...ing__tests__parse_fstring_yield_expr.snap} |   3 +-
 ...ng_parser__tests__parse_empty_fstring.snap |   5 -
 parser/src/string.rs                          | 681 +++++++++++++++++-
 parser/src/string_parser.rs                   | 675 -----------------
 16 files changed, 705 insertions(+), 698 deletions(-)
 rename parser/src/snapshots/{rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap => rustpython_parser__string__tests__fstring_parse_selfdocumenting_base.snap} (96%)
 rename parser/src/snapshots/{rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap => rustpython_parser__string__tests__fstring_parse_selfdocumenting_base_more.snap} (98%)
 rename parser/src/snapshots/{rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap => rustpython_parser__string__tests__fstring_parse_selfdocumenting_format.snap} (98%)
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__parse_empty_fstring.snap
 rename parser/src/snapshots/{rustpython_parser__string_parser__tests__parse_fstring.snap => rustpython_parser__string__tests__parse_fstring.snap} (97%)
 rename parser/src/snapshots/{rustpython_parser__string_parser__tests__parse_fstring_equals.snap => rustpython_parser__string__tests__parse_fstring_equals.snap} (97%)
 rename parser/src/snapshots/{rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap => rustpython_parser__string__tests__parse_fstring_nested_spec.snap} (98%)
 rename parser/src/snapshots/{rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap => rustpython_parser__string__tests__parse_fstring_not_equals.snap} (97%)
 rename parser/src/snapshots/{rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap => rustpython_parser__string__tests__parse_fstring_not_nested_spec.snap} (97%)
 rename parser/src/snapshots/{rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap => rustpython_parser__string__tests__parse_fstring_selfdoc_prec_space.snap} (96%)
 rename parser/src/snapshots/{rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap => rustpython_parser__string__tests__parse_fstring_selfdoc_trailing_space.snap} (96%)
 rename parser/src/snapshots/{rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap => rustpython_parser__string__tests__parse_fstring_yield_expr.snap} (93%)
 delete mode 100644 parser/src/snapshots/rustpython_parser__string_parser__tests__parse_empty_fstring.snap
 delete mode 100644 parser/src/string_parser.rs

diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index ce9dde1d..72dcae6c 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -27,9 +27,8 @@ mod function;
 pub mod lexer;
 pub mod mode;
 pub mod parser;
-mod string_parser;
+mod string;
 #[rustfmt::skip]
 mod python;
 mod context;
-mod string;
 pub mod token;
diff --git a/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap b/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base.snap
similarity index 96%
rename from parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base.snap
index de77be84..cca26adc 100644
--- a/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base.snap
@@ -1,5 +1,6 @@
 ---
-source: compiler/parser/src/string_parser.rs
+source: compiler/parser/src/string.rs
+assertion_line: 698
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap b/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base_more.snap
similarity index 98%
rename from parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base_more.snap
index e7f5bff8..c425d676 100644
--- a/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base_more.snap
@@ -1,5 +1,6 @@
 ---
-source: compiler/parser/src/string_parser.rs
+source: compiler/parser/src/string.rs
+assertion_line: 706
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap b/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_format.snap
similarity index 98%
rename from parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_format.snap
index 5efa6f6c..c402ed68 100644
--- a/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_format.snap
@@ -1,5 +1,6 @@
 ---
-source: compiler/parser/src/string_parser.rs
+source: compiler/parser/src/string.rs
+assertion_line: 714
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_empty_fstring.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_empty_fstring.snap
new file mode 100644
index 00000000..073ac428
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_empty_fstring.snap
@@ -0,0 +1,6 @@
+---
+source: compiler/parser/src/string.rs
+assertion_line: 690
+expression: "parse_fstring(\"\").unwrap()"
+---
+[]
diff --git a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring.snap
similarity index 97%
rename from parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__parse_fstring.snap
index 32b63f95..10b6f3ac 100644
--- a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring.snap
@@ -1,5 +1,6 @@
 ---
-source: compiler/parser/src/string_parser.rs
+source: compiler/parser/src/string.rs
+assertion_line: 669
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_equals.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_equals.snap
similarity index 97%
rename from parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_equals.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_equals.snap
index e9d8120a..194061c3 100644
--- a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_equals.snap
@@ -1,5 +1,6 @@
 ---
-source: compiler/parser/src/string_parser.rs
+source: compiler/parser/src/string.rs
+assertion_line: 766
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_nested_spec.snap
similarity index 98%
rename from parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_nested_spec.snap
index 268a16e9..96ec3e72 100644
--- a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_nested_spec.snap
@@ -1,5 +1,6 @@
 ---
-source: compiler/parser/src/string_parser.rs
+source: compiler/parser/src/string.rs
+assertion_line: 677
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_equals.snap
similarity index 97%
rename from parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_equals.snap
index aaa2ad15..cbfe435d 100644
--- a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_equals.snap
@@ -1,5 +1,6 @@
 ---
-source: compiler/parser/src/string_parser.rs
+source: compiler/parser/src/string.rs
+assertion_line: 759
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_nested_spec.snap
similarity index 97%
rename from parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_nested_spec.snap
index 0d1107ec..2adc5a48 100644
--- a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_nested_spec.snap
@@ -1,5 +1,6 @@
 ---
-source: compiler/parser/src/string_parser.rs
+source: compiler/parser/src/string.rs
+assertion_line: 685
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_prec_space.snap
similarity index 96%
rename from parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_prec_space.snap
index 7851495f..16d62ea1 100644
--- a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_prec_space.snap
@@ -1,5 +1,6 @@
 ---
-source: compiler/parser/src/string_parser.rs
+source: compiler/parser/src/string.rs
+assertion_line: 773
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_trailing_space.snap
similarity index 96%
rename from parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_trailing_space.snap
index 621e4990..16171f2f 100644
--- a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_trailing_space.snap
@@ -1,5 +1,6 @@
 ---
-source: compiler/parser/src/string_parser.rs
+source: compiler/parser/src/string.rs
+assertion_line: 780
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_yield_expr.snap
similarity index 93%
rename from parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_yield_expr.snap
index 62a0cf43..e3a9ff79 100644
--- a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_yield_expr.snap
@@ -1,5 +1,6 @@
 ---
-source: compiler/parser/src/string_parser.rs
+source: compiler/parser/src/string.rs
+assertion_line: 787
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_empty_fstring.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_empty_fstring.snap
deleted file mode 100644
index fbc08b4a..00000000
--- a/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_empty_fstring.snap
+++ /dev/null
@@ -1,5 +0,0 @@
----
-source: compiler/parser/src/string_parser.rs
-expression: "parse_fstring(\"\").unwrap()"
----
-[]
diff --git a/parser/src/string.rs b/parser/src/string.rs
index 027d4a99..84c6fe22 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -1,12 +1,545 @@
+// Contains the logic for parsing string literals (mostly concerned with f-strings.)
+//
+// The lexer doesn't do any special handling of f-strings, it just treats them as
+// regular strings. Since the parser has no definition of f-string formats (Pending PEP 701)
+// we have to do the parsing here, manually.
+use itertools::Itertools;
+
+use self::FStringErrorType::*;
 use crate::{
-    ast::{Constant, Expr, ExprKind, Location},
-    error::{LexicalError, LexicalErrorType},
-    string_parser::parse_string,
+    ast::{Constant, ConversionFlag, Expr, ExprKind, Location},
+    error::{FStringError, FStringErrorType, LexicalError, LexicalErrorType, ParseError},
+    parser::parse_expression_located,
     token::StringKind,
 };
-use itertools::Itertools;
+use std::{iter, str};
+
+// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
+const MAX_UNICODE_NAME: usize = 88;
+
+struct StringParser<'a> {
+    chars: iter::Peekable<str::Chars<'a>>,
+    kind: StringKind,
+    start: Location,
+    end: Location,
+    location: Location,
+}
+
+impl<'a> StringParser<'a> {
+    fn new(
+        source: &'a str,
+        kind: StringKind,
+        triple_quoted: bool,
+        start: Location,
+        end: Location,
+    ) -> Self {
+        let offset = kind.prefix_len() + if triple_quoted { 3 } else { 1 };
+        Self {
+            chars: source.chars().peekable(),
+            kind,
+            start,
+            end,
+            location: start.with_col_offset(offset),
+        }
+    }
+
+    #[inline]
+    fn next_char(&mut self) -> Option<char> {
+        let Some(c) = self.chars.next() else {
+            return None
+        };
+        if c == '\n' {
+            self.location.newline();
+        } else {
+            self.location.go_right();
+        }
+        Some(c)
+    }
+
+    #[inline]
+    fn peek(&mut self) -> Option<&char> {
+        self.chars.peek()
+    }
+
+    #[inline]
+    fn get_pos(&self) -> Location {
+        self.location
+    }
+
+    #[inline]
+    fn expr(&self, node: ExprKind) -> Expr {
+        Expr::new(self.start, self.end, node)
+    }
+
+    fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
+        let mut p: u32 = 0u32;
+        let unicode_error = LexicalError::new(LexicalErrorType::UnicodeError, self.get_pos());
+        for i in 1..=literal_number {
+            match self.next_char() {
+                Some(c) => match c.to_digit(16) {
+                    Some(d) => p += d << ((literal_number - i) * 4),
+                    None => return Err(unicode_error),
+                },
+                None => return Err(unicode_error),
+            }
+        }
+        match p {
+            0xD800..=0xDFFF => Ok(std::char::REPLACEMENT_CHARACTER),
+            _ => std::char::from_u32(p).ok_or(unicode_error),
+        }
+    }
+
+    fn parse_octet(&mut self, first: char) -> char {
+        let mut octet_content = String::new();
+        octet_content.push(first);
+        while octet_content.len() < 3 {
+            if let Some('0'..='7') = self.peek() {
+                octet_content.push(self.next_char().unwrap())
+            } else {
+                break;
+            }
+        }
+        let value = u32::from_str_radix(&octet_content, 8).unwrap();
+        char::from_u32(value).unwrap()
+    }
+
+    fn parse_unicode_name(&mut self) -> Result<char, LexicalError> {
+        let start_pos = self.get_pos();
+        match self.next_char() {
+            Some('{') => {}
+            _ => return Err(LexicalError::new(LexicalErrorType::StringError, start_pos)),
+        }
+        let start_pos = self.get_pos();
+        let mut name = String::new();
+        loop {
+            match self.next_char() {
+                Some('}') => break,
+                Some(c) => name.push(c),
+                None => {
+                    return Err(LexicalError::new(
+                        LexicalErrorType::StringError,
+                        self.get_pos(),
+                    ))
+                }
+            }
+        }
+
+        if name.len() > MAX_UNICODE_NAME {
+            return Err(LexicalError::new(
+                LexicalErrorType::UnicodeError,
+                self.get_pos(),
+            ));
+        }
+
+        unicode_names2::character(&name)
+            .ok_or_else(|| LexicalError::new(LexicalErrorType::UnicodeError, start_pos))
+    }
+
+    fn parse_escaped_char(&mut self) -> Result<String, LexicalError> {
+        match self.next_char() {
+            Some(c) => {
+                let char = match c {
+                    '\\' => '\\',
+                    '\'' => '\'',
+                    '\"' => '"',
+                    'a' => '\x07',
+                    'b' => '\x08',
+                    'f' => '\x0c',
+                    'n' => '\n',
+                    'r' => '\r',
+                    't' => '\t',
+                    'v' => '\x0b',
+                    o @ '0'..='7' => self.parse_octet(o),
+                    'x' => self.parse_unicode_literal(2)?,
+                    'u' if !self.kind.is_bytes() => self.parse_unicode_literal(4)?,
+                    'U' if !self.kind.is_bytes() => self.parse_unicode_literal(8)?,
+                    'N' if !self.kind.is_bytes() => self.parse_unicode_name()?,
+                    // Special cases where the escape sequence is not a single character
+                    '\n' => return Ok("".to_string()),
+                    c => {
+                        if self.kind.is_bytes() && !c.is_ascii() {
+                            return Err(LexicalError {
+                                error: LexicalErrorType::OtherError(
+                                    "bytes can only contain ASCII literal characters".to_owned(),
+                                ),
+                                location: self.get_pos(),
+                            });
+                        }
+                        return Ok(format!("\\{c}"));
+                    }
+                };
+                Ok(char.to_string())
+            }
+            None => Err(LexicalError {
+                error: LexicalErrorType::StringError,
+                location: self.get_pos(),
+            }),
+        }
+    }
+
+    fn parse_formatted_value(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
+        let mut expression = String::new();
+        let mut spec = None;
+        let mut delims = Vec::new();
+        let mut conversion = ConversionFlag::None;
+        let mut self_documenting = false;
+        let mut trailing_seq = String::new();
+        let location = self.get_pos();
+
+        while let Some(ch) = self.next_char() {
+            match ch {
+                // can be integrated better with the remaining code, but as a starting point ok
+                // in general I would do here a tokenizing of the fstrings to omit this peeking.
+                '!' | '=' | '>' | '<' if self.peek() == Some(&'=') => {
+                    expression.push(ch);
+                    expression.push('=');
+                    self.next_char();
+                }
+                '!' if delims.is_empty() && self.peek() != Some(&'=') => {
+                    if expression.trim().is_empty() {
+                        return Err(FStringError::new(EmptyExpression, self.get_pos()).into());
+                    }
+
+                    conversion = match self.next_char() {
+                        Some('s') => ConversionFlag::Str,
+                        Some('a') => ConversionFlag::Ascii,
+                        Some('r') => ConversionFlag::Repr,
+                        Some(_) => {
+                            return Err(
+                                FStringError::new(InvalidConversionFlag, self.get_pos()).into()
+                            );
+                        }
+                        None => {
+                            return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into());
+                        }
+                    };
+
+                    match self.peek() {
+                        Some('}' | ':') => {}
+                        Some(_) | None => {
+                            return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into());
+                        }
+                    }
+                }
+
+                // match a python 3.8 self documenting expression
+                // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}'
+                '=' if self.peek() != Some(&'=') && delims.is_empty() => {
+                    self_documenting = true;
+                }
+
+                ':' if delims.is_empty() => {
+                    let parsed_spec = self.parse_spec(nested)?;
+
+                    spec = Some(Box::new(self.expr(ExprKind::JoinedStr {
+                        values: parsed_spec,
+                    })));
+                }
+                '(' | '{' | '[' => {
+                    expression.push(ch);
+                    delims.push(ch);
+                }
+                ')' => {
+                    let last_delim = delims.pop();
+                    match last_delim {
+                        Some('(') => {
+                            expression.push(ch);
+                        }
+                        Some(c) => {
+                            return Err(FStringError::new(
+                                MismatchedDelimiter(c, ')'),
+                                self.get_pos(),
+                            )
+                            .into());
+                        }
+                        None => {
+                            return Err(FStringError::new(Unmatched(')'), self.get_pos()).into());
+                        }
+                    }
+                }
+                ']' => {
+                    let last_delim = delims.pop();
+                    match last_delim {
+                        Some('[') => {
+                            expression.push(ch);
+                        }
+                        Some(c) => {
+                            return Err(FStringError::new(
+                                MismatchedDelimiter(c, ']'),
+                                self.get_pos(),
+                            )
+                            .into());
+                        }
+                        None => {
+                            return Err(FStringError::new(Unmatched(']'), self.get_pos()).into());
+                        }
+                    }
+                }
+                '}' if !delims.is_empty() => {
+                    let last_delim = delims.pop();
+                    match last_delim {
+                        Some('{') => {
+                            expression.push(ch);
+                        }
+                        Some(c) => {
+                            return Err(FStringError::new(
+                                MismatchedDelimiter(c, '}'),
+                                self.get_pos(),
+                            )
+                            .into());
+                        }
+                        None => {}
+                    }
+                }
+                '}' => {
+                    if expression.trim().is_empty() {
+                        return Err(FStringError::new(EmptyExpression, self.get_pos()).into());
+                    }
+
+                    let ret = if !self_documenting {
+                        vec![self.expr(ExprKind::FormattedValue {
+                            value: Box::new(parse_fstring_expr(&expression, location).map_err(
+                                |e| {
+                                    FStringError::new(
+                                        InvalidExpression(Box::new(e.error)),
+                                        location,
+                                    )
+                                },
+                            )?),
+                            conversion: conversion as _,
+                            format_spec: spec,
+                        })]
+                    } else {
+                        vec![
+                            self.expr(ExprKind::Constant {
+                                value: Constant::Str(expression.to_owned() + "="),
+                                kind: None,
+                            }),
+                            self.expr(ExprKind::Constant {
+                                value: trailing_seq.into(),
+                                kind: None,
+                            }),
+                            self.expr(ExprKind::FormattedValue {
+                                value: Box::new(
+                                    parse_fstring_expr(&expression, location).map_err(|e| {
+                                        FStringError::new(
+                                            InvalidExpression(Box::new(e.error)),
+                                            location,
+                                        )
+                                    })?,
+                                ),
+                                conversion: (if conversion == ConversionFlag::None && spec.is_none()
+                                {
+                                    ConversionFlag::Repr
+                                } else {
+                                    conversion
+                                }) as _,
+                                format_spec: spec,
+                            }),
+                        ]
+                    };
+                    return Ok(ret);
+                }
+                '"' | '\'' => {
+                    expression.push(ch);
+                    loop {
+                        let Some(c) = self.next_char() else {
+                            return Err(FStringError::new(UnterminatedString, self.get_pos()).into());
+                        };
+                        expression.push(c);
+                        if c == ch {
+                            break;
+                        }
+                    }
+                }
+                ' ' if self_documenting => {
+                    trailing_seq.push(ch);
+                }
+                '\\' => return Err(FStringError::new(UnterminatedString, self.get_pos()).into()),
+                _ => {
+                    if self_documenting {
+                        return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into());
+                    }
+
+                    expression.push(ch);
+                }
+            }
+        }
+        Err(FStringError::new(UnclosedLbrace, self.get_pos()).into())
+    }
+
+    fn parse_spec(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
+        let mut spec_constructor = Vec::new();
+        let mut constant_piece = String::new();
+        while let Some(&next) = self.peek() {
+            match next {
+                '{' => {
+                    if !constant_piece.is_empty() {
+                        spec_constructor.push(self.expr(ExprKind::Constant {
+                            value: constant_piece.drain(..).collect::<String>().into(),
+                            kind: None,
+                        }));
+                    }
+                    let parsed_expr = self.parse_fstring(nested + 1)?;
+                    spec_constructor.extend(parsed_expr);
+                    continue;
+                }
+                '}' => {
+                    break;
+                }
+                _ => {
+                    constant_piece.push(next);
+                }
+            }
+            self.next_char();
+        }
+        if !constant_piece.is_empty() {
+            spec_constructor.push(self.expr(ExprKind::Constant {
+                value: constant_piece.drain(..).collect::<String>().into(),
+                kind: None,
+            }));
+        }
+        Ok(spec_constructor)
+    }
+
+    fn parse_fstring(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
+        if nested >= 2 {
+            return Err(FStringError::new(ExpressionNestedTooDeeply, self.get_pos()).into());
+        }
+
+        let mut content = String::new();
+        let mut values = vec![];
+
+        while let Some(&ch) = self.peek() {
+            match ch {
+                '{' => {
+                    self.next_char();
+                    if nested == 0 {
+                        match self.peek() {
+                            Some('{') => {
+                                self.next_char();
+                                content.push('{');
+                                continue;
+                            }
+                            None => {
+                                return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into())
+                            }
+                            _ => {}
+                        }
+                    }
+                    if !content.is_empty() {
+                        values.push(self.expr(ExprKind::Constant {
+                            value: content.drain(..).collect::<String>().into(),
+                            kind: None,
+                        }));
+                    }
+
+                    let parsed_values = self.parse_formatted_value(nested)?;
+                    values.extend(parsed_values);
+                }
+                '}' => {
+                    if nested > 0 {
+                        break;
+                    }
+                    self.next_char();
+                    if let Some('}') = self.peek() {
+                        self.next_char();
+                        content.push('}');
+                    } else {
+                        return Err(FStringError::new(SingleRbrace, self.get_pos()).into());
+                    }
+                }
+                '\\' if !self.kind.is_raw() => {
+                    self.next_char();
+                    content.push_str(&self.parse_escaped_char()?);
+                }
+                _ => {
+                    content.push(ch);
+                    self.next_char();
+                }
+            }
+        }
+
+        if !content.is_empty() {
+            values.push(self.expr(ExprKind::Constant {
+                value: content.into(),
+                kind: None,
+            }))
+        }
+
+        Ok(values)
+    }
+
+    fn parse_bytes(&mut self) -> Result<Expr, LexicalError> {
+        let mut content = String::new();
+        while let Some(ch) = self.next_char() {
+            match ch {
+                '\\' if !self.kind.is_raw() => {
+                    content.push_str(&self.parse_escaped_char()?);
+                }
+                ch => {
+                    if !ch.is_ascii() {
+                        return Err(LexicalError::new(
+                            LexicalErrorType::OtherError(
+                                "bytes can only contain ASCII literal characters".to_string(),
+                            ),
+                            self.get_pos(),
+                        ));
+                    }
+                    content.push(ch);
+                }
+            }
+        }
+
+        Ok(self.expr(ExprKind::Constant {
+            value: Constant::Bytes(content.chars().map(|c| c as u8).collect()),
+            kind: None,
+        }))
+    }
+
+    fn parse_string(&mut self) -> Result<Expr, LexicalError> {
+        let mut content = String::new();
+        while let Some(ch) = self.next_char() {
+            match ch {
+                '\\' if !self.kind.is_raw() => {
+                    content.push_str(&self.parse_escaped_char()?);
+                }
+                ch => content.push(ch),
+            }
+        }
+        Ok(self.expr(ExprKind::Constant {
+            value: Constant::Str(content),
+            kind: self.kind.is_unicode().then(|| "u".to_string()),
+        }))
+    }
+
+    fn parse(&mut self) -> Result<Vec<Expr>, LexicalError> {
+        if self.kind.is_fstring() {
+            self.parse_fstring(0)
+        } else if self.kind.is_bytes() {
+            self.parse_bytes().map(|expr| vec![expr])
+        } else {
+            self.parse_string().map(|expr| vec![expr])
+        }
+    }
+}
+
+fn parse_fstring_expr(source: &str, location: Location) -> Result<Expr, ParseError> {
+    let fstring_body = format!("({source})");
+    parse_expression_located(&fstring_body, "<fstring>", location.with_col_offset(-1))
+}
 
-pub fn parse_strings(
+fn parse_string(
+    source: &str,
+    kind: StringKind,
+    triple_quoted: bool,
+    start: Location,
+    end: Location,
+) -> Result<Vec<Expr>, LexicalError> {
+    StringParser::new(source, kind, triple_quoted, start, end).parse()
+}
+
+pub(crate) fn parse_strings(
     values: Vec<(Location, (String, StringKind, bool), Location)>,
 ) -> Result<Expr, LexicalError> {
     // Preserve the initial location and kind.
@@ -120,8 +653,146 @@ pub fn parse_strings(
 
 #[cfg(test)]
 mod tests {
+    use super::*;
     use crate::parser::parse_program;
 
+    fn parse_fstring(source: &str) -> Result<Vec<Expr>, LexicalError> {
+        StringParser::new(
+            source,
+            StringKind::FString,
+            false,
+            Location::default(),
+            Location::default().with_col_offset(source.len() + 3), // 3 for prefix and quotes
+        )
+        .parse()
+    }
+
+    #[test]
+    fn test_parse_fstring() {
+        let source = "{a}{ b }{{foo}}";
+        let parse_ast = parse_fstring(source).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_nested_spec() {
+        let source = "{foo:{spec}}";
+        let parse_ast = parse_fstring(source).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_not_nested_spec() {
+        let source = "{foo:spec}";
+        let parse_ast = parse_fstring(source).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_empty_fstring() {
+        insta::assert_debug_snapshot!(parse_fstring("").unwrap());
+    }
+
+    #[test]
+    fn test_fstring_parse_selfdocumenting_base() {
+        let src = "{user=}";
+        let parse_ast = parse_fstring(src).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_fstring_parse_selfdocumenting_base_more() {
+        let src = "mix {user=} with text and {second=}";
+        let parse_ast = parse_fstring(src).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_fstring_parse_selfdocumenting_format() {
+        let src = "{user=:>10}";
+        let parse_ast = parse_fstring(src).unwrap();
+
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    fn parse_fstring_error(source: &str) -> FStringErrorType {
+        parse_fstring(source)
+            .map_err(|e| match e.error {
+                LexicalErrorType::FStringError(e) => e,
+                e => unreachable!("Expected FStringError: {:?}", e),
+            })
+            .err()
+            .expect("Expected error")
+    }
+
+    #[test]
+    fn test_parse_invalid_fstring() {
+        assert_eq!(parse_fstring_error("{5!a"), UnclosedLbrace);
+        assert_eq!(parse_fstring_error("{5!a1}"), UnclosedLbrace);
+        assert_eq!(parse_fstring_error("{5!"), UnclosedLbrace);
+        assert_eq!(parse_fstring_error("abc{!a 'cat'}"), EmptyExpression);
+        assert_eq!(parse_fstring_error("{!a"), EmptyExpression);
+        assert_eq!(parse_fstring_error("{ !a}"), EmptyExpression);
+
+        assert_eq!(parse_fstring_error("{5!}"), InvalidConversionFlag);
+        assert_eq!(parse_fstring_error("{5!x}"), InvalidConversionFlag);
+
+        assert_eq!(
+            parse_fstring_error("{a:{a:{b}}}"),
+            ExpressionNestedTooDeeply
+        );
+
+        assert_eq!(parse_fstring_error("{a:b}}"), SingleRbrace);
+        assert_eq!(parse_fstring_error("}"), SingleRbrace);
+        assert_eq!(parse_fstring_error("{a:{b}"), UnclosedLbrace);
+        assert_eq!(parse_fstring_error("{"), UnclosedLbrace);
+
+        assert_eq!(parse_fstring_error("{}"), EmptyExpression);
+
+        // TODO: check for InvalidExpression enum?
+        assert!(parse_fstring("{class}").is_err());
+    }
+
+    #[test]
+    fn test_parse_fstring_not_equals() {
+        let source = "{1 != 2}";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_equals() {
+        let source = "{42 == 42}";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_selfdoc_prec_space() {
+        let source = "{x   =}";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_selfdoc_trailing_space() {
+        let source = "{x=   }";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_parse_fstring_yield_expr() {
+        let source = "{yield}";
+        let parse_ast = parse_fstring(source).unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
     #[test]
     fn test_parse_string_concat() {
         let source = "'Hello ' 'world'";
diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs
deleted file mode 100644
index c18ba0f9..00000000
--- a/parser/src/string_parser.rs
+++ /dev/null
@@ -1,675 +0,0 @@
-use self::FStringErrorType::*;
-use crate::{
-    ast::{Constant, ConversionFlag, Expr, ExprKind, Location},
-    error::{FStringError, FStringErrorType, LexicalError, LexicalErrorType, ParseError},
-    parser::parse_expression_located,
-    token::StringKind,
-};
-use std::{iter, str};
-
-/// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
-pub const MAX_UNICODE_NAME: usize = 88;
-
-pub struct StringParser<'a> {
-    chars: iter::Peekable<str::Chars<'a>>,
-    kind: StringKind,
-    str_start: Location,
-    str_end: Location,
-    location: Location,
-}
-
-impl<'a> StringParser<'a> {
-    pub fn new(
-        source: &'a str,
-        kind: StringKind,
-        triple_quoted: bool,
-        str_start: Location,
-        str_end: Location,
-    ) -> Self {
-        let offset = kind.prefix_len() + if triple_quoted { 3 } else { 1 };
-        Self {
-            chars: source.chars().peekable(),
-            kind,
-            str_start,
-            str_end,
-            location: str_start.with_col_offset(offset),
-        }
-    }
-
-    #[inline]
-    fn next_char(&mut self) -> Option<char> {
-        let Some(c) = self.chars.next() else {
-            return None
-        };
-        if c == '\n' {
-            self.location.newline();
-        } else {
-            self.location.go_right();
-        }
-        Some(c)
-    }
-
-    #[inline]
-    fn peek(&mut self) -> Option<&char> {
-        self.chars.peek()
-    }
-
-    #[inline]
-    fn get_pos(&self) -> Location {
-        self.location
-    }
-
-    #[inline]
-    fn expr(&self, node: ExprKind) -> Expr {
-        Expr::new(self.str_start, self.str_end, node)
-    }
-
-    fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
-        let mut p: u32 = 0u32;
-        let unicode_error = LexicalError::new(LexicalErrorType::UnicodeError, self.get_pos());
-        for i in 1..=literal_number {
-            match self.next_char() {
-                Some(c) => match c.to_digit(16) {
-                    Some(d) => p += d << ((literal_number - i) * 4),
-                    None => return Err(unicode_error),
-                },
-                None => return Err(unicode_error),
-            }
-        }
-        match p {
-            0xD800..=0xDFFF => Ok(std::char::REPLACEMENT_CHARACTER),
-            _ => std::char::from_u32(p).ok_or(unicode_error),
-        }
-    }
-
-    fn parse_octet(&mut self, first: char) -> char {
-        let mut octet_content = String::new();
-        octet_content.push(first);
-        while octet_content.len() < 3 {
-            if let Some('0'..='7') = self.peek() {
-                octet_content.push(self.next_char().unwrap())
-            } else {
-                break;
-            }
-        }
-        let value = u32::from_str_radix(&octet_content, 8).unwrap();
-        char::from_u32(value).unwrap()
-    }
-
-    fn parse_unicode_name(&mut self) -> Result<char, LexicalError> {
-        let start_pos = self.get_pos();
-        match self.next_char() {
-            Some('{') => {}
-            _ => return Err(LexicalError::new(LexicalErrorType::StringError, start_pos)),
-        }
-        let start_pos = self.get_pos();
-        let mut name = String::new();
-        loop {
-            match self.next_char() {
-                Some('}') => break,
-                Some(c) => name.push(c),
-                None => {
-                    return Err(LexicalError::new(
-                        LexicalErrorType::StringError,
-                        self.get_pos(),
-                    ))
-                }
-            }
-        }
-
-        if name.len() > MAX_UNICODE_NAME {
-            return Err(LexicalError::new(
-                LexicalErrorType::UnicodeError,
-                self.get_pos(),
-            ));
-        }
-
-        unicode_names2::character(&name)
-            .ok_or_else(|| LexicalError::new(LexicalErrorType::UnicodeError, start_pos))
-    }
-
-    fn parse_escaped_char(&mut self) -> Result<String, LexicalError> {
-        match self.next_char() {
-            Some(c) => {
-                let char = match c {
-                    '\\' => '\\',
-                    '\'' => '\'',
-                    '\"' => '"',
-                    'a' => '\x07',
-                    'b' => '\x08',
-                    'f' => '\x0c',
-                    'n' => '\n',
-                    'r' => '\r',
-                    't' => '\t',
-                    'v' => '\x0b',
-                    o @ '0'..='7' => self.parse_octet(o),
-                    'x' => self.parse_unicode_literal(2)?,
-                    'u' if !self.kind.is_bytes() => self.parse_unicode_literal(4)?,
-                    'U' if !self.kind.is_bytes() => self.parse_unicode_literal(8)?,
-                    'N' if !self.kind.is_bytes() => self.parse_unicode_name()?,
-                    // Special cases where the escape sequence is not a single character
-                    '\n' => return Ok("".to_string()),
-                    c => {
-                        if self.kind.is_bytes() && !c.is_ascii() {
-                            return Err(LexicalError {
-                                error: LexicalErrorType::OtherError(
-                                    "bytes can only contain ASCII literal characters".to_owned(),
-                                ),
-                                location: self.get_pos(),
-                            });
-                        }
-                        return Ok(format!("\\{c}"));
-                    }
-                };
-                Ok(char.to_string())
-            }
-            None => Err(LexicalError {
-                error: LexicalErrorType::StringError,
-                location: self.get_pos(),
-            }),
-        }
-    }
-
-    fn parse_formatted_value(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
-        let mut expression = String::new();
-        let mut spec = None;
-        let mut delims = Vec::new();
-        let mut conversion = ConversionFlag::None;
-        let mut self_documenting = false;
-        let mut trailing_seq = String::new();
-        let location = self.get_pos();
-
-        while let Some(ch) = self.next_char() {
-            match ch {
-                // can be integrated better with the remaining code, but as a starting point ok
-                // in general I would do here a tokenizing of the fstrings to omit this peeking.
-                '!' | '=' | '>' | '<' if self.peek() == Some(&'=') => {
-                    expression.push(ch);
-                    expression.push('=');
-                    self.next_char();
-                }
-                '!' if delims.is_empty() && self.peek() != Some(&'=') => {
-                    if expression.trim().is_empty() {
-                        return Err(FStringError::new(EmptyExpression, self.get_pos()).into());
-                    }
-
-                    conversion = match self.next_char() {
-                        Some('s') => ConversionFlag::Str,
-                        Some('a') => ConversionFlag::Ascii,
-                        Some('r') => ConversionFlag::Repr,
-                        Some(_) => {
-                            return Err(
-                                FStringError::new(InvalidConversionFlag, self.get_pos()).into()
-                            );
-                        }
-                        None => {
-                            return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into());
-                        }
-                    };
-
-                    match self.peek() {
-                        Some('}' | ':') => {}
-                        Some(_) | None => {
-                            return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into());
-                        }
-                    }
-                }
-
-                // match a python 3.8 self documenting expression
-                // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}'
-                '=' if self.peek() != Some(&'=') && delims.is_empty() => {
-                    self_documenting = true;
-                }
-
-                ':' if delims.is_empty() => {
-                    let parsed_spec = self.parse_spec(nested)?;
-
-                    spec = Some(Box::new(self.expr(ExprKind::JoinedStr {
-                        values: parsed_spec,
-                    })));
-                }
-                '(' | '{' | '[' => {
-                    expression.push(ch);
-                    delims.push(ch);
-                }
-                ')' => {
-                    let last_delim = delims.pop();
-                    match last_delim {
-                        Some('(') => {
-                            expression.push(ch);
-                        }
-                        Some(c) => {
-                            return Err(FStringError::new(
-                                MismatchedDelimiter(c, ')'),
-                                self.get_pos(),
-                            )
-                            .into());
-                        }
-                        None => {
-                            return Err(FStringError::new(Unmatched(')'), self.get_pos()).into());
-                        }
-                    }
-                }
-                ']' => {
-                    let last_delim = delims.pop();
-                    match last_delim {
-                        Some('[') => {
-                            expression.push(ch);
-                        }
-                        Some(c) => {
-                            return Err(FStringError::new(
-                                MismatchedDelimiter(c, ']'),
-                                self.get_pos(),
-                            )
-                            .into());
-                        }
-                        None => {
-                            return Err(FStringError::new(Unmatched(']'), self.get_pos()).into());
-                        }
-                    }
-                }
-                '}' if !delims.is_empty() => {
-                    let last_delim = delims.pop();
-                    match last_delim {
-                        Some('{') => {
-                            expression.push(ch);
-                        }
-                        Some(c) => {
-                            return Err(FStringError::new(
-                                MismatchedDelimiter(c, '}'),
-                                self.get_pos(),
-                            )
-                            .into());
-                        }
-                        None => {}
-                    }
-                }
-                '}' => {
-                    if expression.trim().is_empty() {
-                        return Err(FStringError::new(EmptyExpression, self.get_pos()).into());
-                    }
-
-                    let ret = if !self_documenting {
-                        vec![self.expr(ExprKind::FormattedValue {
-                            value: Box::new(parse_fstring_expr(&expression, location).map_err(
-                                |e| {
-                                    FStringError::new(
-                                        InvalidExpression(Box::new(e.error)),
-                                        location,
-                                    )
-                                },
-                            )?),
-                            conversion: conversion as _,
-                            format_spec: spec,
-                        })]
-                    } else {
-                        vec![
-                            self.expr(ExprKind::Constant {
-                                value: Constant::Str(expression.to_owned() + "="),
-                                kind: None,
-                            }),
-                            self.expr(ExprKind::Constant {
-                                value: trailing_seq.into(),
-                                kind: None,
-                            }),
-                            self.expr(ExprKind::FormattedValue {
-                                value: Box::new(
-                                    parse_fstring_expr(&expression, location).map_err(|e| {
-                                        FStringError::new(
-                                            InvalidExpression(Box::new(e.error)),
-                                            location,
-                                        )
-                                    })?,
-                                ),
-                                conversion: (if conversion == ConversionFlag::None && spec.is_none()
-                                {
-                                    ConversionFlag::Repr
-                                } else {
-                                    conversion
-                                }) as _,
-                                format_spec: spec,
-                            }),
-                        ]
-                    };
-                    return Ok(ret);
-                }
-                '"' | '\'' => {
-                    expression.push(ch);
-                    loop {
-                        let Some(c) = self.next_char() else {
-                            return Err(FStringError::new(UnterminatedString, self.get_pos()).into());
-                        };
-                        expression.push(c);
-                        if c == ch {
-                            break;
-                        }
-                    }
-                }
-                ' ' if self_documenting => {
-                    trailing_seq.push(ch);
-                }
-                '\\' => return Err(FStringError::new(UnterminatedString, self.get_pos()).into()),
-                _ => {
-                    if self_documenting {
-                        return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into());
-                    }
-
-                    expression.push(ch);
-                }
-            }
-        }
-        Err(FStringError::new(UnclosedLbrace, self.get_pos()).into())
-    }
-
-    fn parse_spec(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
-        let mut spec_constructor = Vec::new();
-        let mut constant_piece = String::new();
-        while let Some(&next) = self.peek() {
-            match next {
-                '{' => {
-                    if !constant_piece.is_empty() {
-                        spec_constructor.push(self.expr(ExprKind::Constant {
-                            value: constant_piece.drain(..).collect::<String>().into(),
-                            kind: None,
-                        }));
-                    }
-                    let parsed_expr = self.parse_fstring(nested + 1)?;
-                    spec_constructor.extend(parsed_expr);
-                    continue;
-                }
-                '}' => {
-                    break;
-                }
-                _ => {
-                    constant_piece.push(next);
-                }
-            }
-            self.next_char();
-        }
-        if !constant_piece.is_empty() {
-            spec_constructor.push(self.expr(ExprKind::Constant {
-                value: constant_piece.drain(..).collect::<String>().into(),
-                kind: None,
-            }));
-        }
-        Ok(spec_constructor)
-    }
-
-    fn parse_fstring(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
-        if nested >= 2 {
-            return Err(FStringError::new(ExpressionNestedTooDeeply, self.get_pos()).into());
-        }
-
-        let mut content = String::new();
-        let mut values = vec![];
-
-        while let Some(&ch) = self.peek() {
-            match ch {
-                '{' => {
-                    self.next_char();
-                    if nested == 0 {
-                        match self.peek() {
-                            Some('{') => {
-                                self.next_char();
-                                content.push('{');
-                                continue;
-                            }
-                            None => {
-                                return Err(FStringError::new(UnclosedLbrace, self.get_pos()).into())
-                            }
-                            _ => {}
-                        }
-                    }
-                    if !content.is_empty() {
-                        values.push(self.expr(ExprKind::Constant {
-                            value: content.drain(..).collect::<String>().into(),
-                            kind: None,
-                        }));
-                    }
-
-                    let parsed_values = self.parse_formatted_value(nested)?;
-                    values.extend(parsed_values);
-                }
-                '}' => {
-                    if nested > 0 {
-                        break;
-                    }
-                    self.next_char();
-                    if let Some('}') = self.peek() {
-                        self.next_char();
-                        content.push('}');
-                    } else {
-                        return Err(FStringError::new(SingleRbrace, self.get_pos()).into());
-                    }
-                }
-                '\\' if !self.kind.is_raw() => {
-                    self.next_char();
-                    content.push_str(&self.parse_escaped_char()?);
-                }
-                _ => {
-                    content.push(ch);
-                    self.next_char();
-                }
-            }
-        }
-
-        if !content.is_empty() {
-            values.push(self.expr(ExprKind::Constant {
-                value: content.into(),
-                kind: None,
-            }))
-        }
-
-        Ok(values)
-    }
-
-    pub fn parse_bytes(&mut self) -> Result<Expr, LexicalError> {
-        let mut content = String::new();
-        while let Some(ch) = self.next_char() {
-            match ch {
-                '\\' if !self.kind.is_raw() => {
-                    content.push_str(&self.parse_escaped_char()?);
-                }
-                ch => {
-                    if !ch.is_ascii() {
-                        return Err(LexicalError::new(
-                            LexicalErrorType::OtherError(
-                                "bytes can only contain ASCII literal characters".to_string(),
-                            ),
-                            self.get_pos(),
-                        ));
-                    }
-                    content.push(ch);
-                }
-            }
-        }
-
-        Ok(self.expr(ExprKind::Constant {
-            value: Constant::Bytes(content.chars().map(|c| c as u8).collect()),
-            kind: None,
-        }))
-    }
-
-    pub fn parse_string(&mut self) -> Result<Expr, LexicalError> {
-        let mut content = String::new();
-        while let Some(ch) = self.next_char() {
-            match ch {
-                '\\' if !self.kind.is_raw() => {
-                    content.push_str(&self.parse_escaped_char()?);
-                }
-                ch => content.push(ch),
-            }
-        }
-        Ok(self.expr(ExprKind::Constant {
-            value: Constant::Str(content),
-            kind: self.kind.is_unicode().then(|| "u".to_string()),
-        }))
-    }
-
-    pub fn parse(&mut self) -> Result<Vec<Expr>, LexicalError> {
-        if self.kind.is_fstring() {
-            self.parse_fstring(0)
-        } else if self.kind.is_bytes() {
-            self.parse_bytes().map(|expr| vec![expr])
-        } else {
-            self.parse_string().map(|expr| vec![expr])
-        }
-    }
-}
-
-fn parse_fstring_expr(source: &str, location: Location) -> Result<Expr, ParseError> {
-    let fstring_body = format!("({source})");
-    parse_expression_located(&fstring_body, "<fstring>", location.with_col_offset(-1))
-}
-
-pub fn parse_string(
-    source: &str,
-    kind: StringKind,
-    triple_quoted: bool,
-    start: Location,
-    end: Location,
-) -> Result<Vec<Expr>, LexicalError> {
-    StringParser::new(source, kind, triple_quoted, start, end).parse()
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn parse_fstring(source: &str) -> Result<Vec<Expr>, LexicalError> {
-        StringParser::new(
-            source,
-            StringKind::FString,
-            false,
-            Location::default(),
-            Location::default().with_col_offset(source.len() + 3), // 3 for prefix and quotes
-        )
-        .parse()
-    }
-
-    #[test]
-    fn test_parse_fstring() {
-        let source = "{a}{ b }{{foo}}";
-        let parse_ast = parse_fstring(source).unwrap();
-
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_fstring_nested_spec() {
-        let source = "{foo:{spec}}";
-        let parse_ast = parse_fstring(source).unwrap();
-
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_fstring_not_nested_spec() {
-        let source = "{foo:spec}";
-        let parse_ast = parse_fstring(source).unwrap();
-
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_empty_fstring() {
-        insta::assert_debug_snapshot!(parse_fstring("").unwrap());
-    }
-
-    #[test]
-    fn test_fstring_parse_selfdocumenting_base() {
-        let src = "{user=}";
-        let parse_ast = parse_fstring(src).unwrap();
-
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_fstring_parse_selfdocumenting_base_more() {
-        let src = "mix {user=} with text and {second=}";
-        let parse_ast = parse_fstring(src).unwrap();
-
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_fstring_parse_selfdocumenting_format() {
-        let src = "{user=:>10}";
-        let parse_ast = parse_fstring(src).unwrap();
-
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    fn parse_fstring_error(source: &str) -> FStringErrorType {
-        parse_fstring(source)
-            .map_err(|e| match e.error {
-                LexicalErrorType::FStringError(e) => e,
-                e => unreachable!("Expected FStringError: {:?}", e),
-            })
-            .err()
-            .expect("Expected error")
-    }
-
-    #[test]
-    fn test_parse_invalid_fstring() {
-        assert_eq!(parse_fstring_error("{5!a"), UnclosedLbrace);
-        assert_eq!(parse_fstring_error("{5!a1}"), UnclosedLbrace);
-        assert_eq!(parse_fstring_error("{5!"), UnclosedLbrace);
-        assert_eq!(parse_fstring_error("abc{!a 'cat'}"), EmptyExpression);
-        assert_eq!(parse_fstring_error("{!a"), EmptyExpression);
-        assert_eq!(parse_fstring_error("{ !a}"), EmptyExpression);
-
-        assert_eq!(parse_fstring_error("{5!}"), InvalidConversionFlag);
-        assert_eq!(parse_fstring_error("{5!x}"), InvalidConversionFlag);
-
-        assert_eq!(
-            parse_fstring_error("{a:{a:{b}}}"),
-            ExpressionNestedTooDeeply
-        );
-
-        assert_eq!(parse_fstring_error("{a:b}}"), SingleRbrace);
-        assert_eq!(parse_fstring_error("}"), SingleRbrace);
-        assert_eq!(parse_fstring_error("{a:{b}"), UnclosedLbrace);
-        assert_eq!(parse_fstring_error("{"), UnclosedLbrace);
-
-        assert_eq!(parse_fstring_error("{}"), EmptyExpression);
-
-        // TODO: check for InvalidExpression enum?
-        assert!(parse_fstring("{class}").is_err());
-    }
-
-    #[test]
-    fn test_parse_fstring_not_equals() {
-        let source = "{1 != 2}";
-        let parse_ast = parse_fstring(source).unwrap();
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_fstring_equals() {
-        let source = "{42 == 42}";
-        let parse_ast = parse_fstring(source).unwrap();
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_fstring_selfdoc_prec_space() {
-        let source = "{x   =}";
-        let parse_ast = parse_fstring(source).unwrap();
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_fstring_selfdoc_trailing_space() {
-        let source = "{x=   }";
-        let parse_ast = parse_fstring(source).unwrap();
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-
-    #[test]
-    fn test_parse_fstring_yield_expr() {
-        let source = "{yield}";
-        let parse_ast = parse_fstring(source).unwrap();
-        insta::assert_debug_snapshot!(parse_ast);
-    }
-}

From 7714c5f5172506243e4ed2e23708e484a6602fdd Mon Sep 17 00:00:00 2001
From: Dimitris Fasarakis Hilliard <d.f.hilliard@gmail.com>
Date: Sat, 11 Feb 2023 23:03:29 +0200
Subject: [PATCH 415/459] Add tests, some comments, to function.rs.

---
 parser/src/function.rs                        | 104 +++++++-
 ...unction__tests__function_kw_only_args.snap | 108 ++++++++
 ...__function_kw_only_args_with_defaults.snap | 147 +++++++++++
 ...er__function__tests__function_no_args.snap |  52 ++++
 ..._tests__function_pos_and_kw_only_args.snap | 162 ++++++++++++
 ...on_pos_and_kw_only_args_with_defaults.snap | 201 +++++++++++++++
 ...w_only_args_with_defaults_and_varargs.snap | 220 ++++++++++++++++
 ..._with_defaults_and_varargs_and_kwargs.snap | 239 ++++++++++++++++++
 ...r__function__tests__function_pos_args.snap | 107 ++++++++
 ...ests__function_pos_args_with_defaults.snap | 146 +++++++++++
 ..._function__tests__lambda_kw_only_args.snap | 121 +++++++++
 ...ts__lambda_kw_only_args_with_defaults.snap | 160 ++++++++++++
 ...rser__function__tests__lambda_no_args.snap |  66 +++++
 ...n__tests__lambda_pos_and_kw_only_args.snap | 158 ++++++++++++
 ...ser__function__tests__lambda_pos_args.snap | 121 +++++++++
 ..._tests__lambda_pos_args_with_defaults.snap | 160 ++++++++++++
 16 files changed, 2265 insertions(+), 7 deletions(-)
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args_with_defaults.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__function_no_args.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs_and_kwargs.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__function_pos_args.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__function_pos_args_with_defaults.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args_with_defaults.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__lambda_no_args.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_and_kw_only_args.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args_with_defaults.snap

diff --git a/parser/src/function.rs b/parser/src/function.rs
index 21ccf013..35c3ad29 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -1,8 +1,10 @@
+// Contains functions that perform validation and parsing of arguments and parameters.
+// Checks apply both to functions and to lambdas.
 use crate::ast;
 use crate::error::{LexicalError, LexicalErrorType};
 use rustc_hash::FxHashSet;
 
-pub struct ArgumentList {
+pub(crate) struct ArgumentList {
     pub args: Vec<ast::Expr>,
     pub keywords: Vec<ast::Keyword>,
 }
@@ -10,7 +12,10 @@ pub struct ArgumentList {
 type ParameterDefs = (Vec<ast::Arg>, Vec<ast::Arg>, Vec<ast::Expr>);
 type ParameterDef = (ast::Arg, Option<ast::Expr>);
 
-pub fn validate_arguments(arguments: ast::Arguments) -> Result<ast::Arguments, LexicalError> {
+// Perform validation of function/lambda arguments in a function definition.
+pub(crate) fn validate_arguments(
+    arguments: ast::Arguments,
+) -> Result<ast::Arguments, LexicalError> {
     let mut all_args: Vec<&ast::Located<ast::ArgData>> = vec![];
 
     all_args.extend(arguments.posonlyargs.iter());
@@ -29,6 +34,7 @@ pub fn validate_arguments(arguments: ast::Arguments) -> Result<ast::Arguments, L
     let mut all_arg_names = FxHashSet::with_hasher(Default::default());
     for arg in all_args {
         let arg_name = &arg.node.arg;
+        // Check for duplicate arguments in the function definition.
         if !all_arg_names.insert(arg_name) {
             return Err(LexicalError {
                 error: LexicalErrorType::DuplicateArgumentError(arg_name.to_string()),
@@ -40,7 +46,8 @@ pub fn validate_arguments(arguments: ast::Arguments) -> Result<ast::Arguments, L
     Ok(arguments)
 }
 
-pub fn parse_params(
+// Parse parameters as supplied during a function/lambda *definition*.
+pub(crate) fn parse_params(
     params: (Vec<ParameterDef>, Vec<ParameterDef>),
 ) -> Result<ParameterDefs, LexicalError> {
     let mut pos_only = Vec::with_capacity(params.0.len());
@@ -52,7 +59,7 @@ pub fn parse_params(
             defaults.push(default);
         } else if !defaults.is_empty() {
             // Once we have started with defaults, all remaining arguments must
-            // have defaults
+            // have defaults.
             return Err(LexicalError {
                 error: LexicalErrorType::DefaultArgumentError,
                 location: name.location,
@@ -79,7 +86,8 @@ type FunctionArgument = (
     ast::Expr,
 );
 
-pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, LexicalError> {
+// Parse arguments as supplied during a function/lambda *call*.
+pub(crate) fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, LexicalError> {
     let mut args = vec![];
     let mut keywords = vec![];
 
@@ -89,6 +97,7 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, Lexi
     for (name, value) in func_args {
         match name {
             Some((start, end, name)) => {
+                // Check for duplicate keyword arguments in the call.
                 if let Some(keyword_name) = &name {
                     if keyword_names.contains(keyword_name) {
                         return Err(LexicalError {
@@ -111,13 +120,14 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, Lexi
                 ));
             }
             None => {
-                // Allow starred arguments after keyword arguments but
-                // not after double-starred arguments.
+                // Positional arguments mustn't follow keyword arguments.
                 if !keywords.is_empty() && !is_starred(&value) {
                     return Err(LexicalError {
                         error: LexicalErrorType::PositionalArgumentError,
                         location: value.location,
                     });
+                // Allow starred arguments after keyword arguments but
+                // not after double-starred arguments.
                 } else if double_starred {
                     return Err(LexicalError {
                         error: LexicalErrorType::UnpackedArgumentError,
@@ -132,6 +142,86 @@ pub fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, Lexi
     Ok(ArgumentList { args, keywords })
 }
 
+// Check if an expression is a starred expression.
 fn is_starred(exp: &ast::Expr) -> bool {
     matches!(exp.node, ast::ExprKind::Starred { .. })
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::error::{LexicalErrorType, ParseErrorType};
+    use crate::parser::parse_program;
+
+    macro_rules! function_and_lambda {
+        ($($name:ident: $code:expr,)*) => {
+            $(
+                #[test]
+                fn $name() {
+                    let parse_ast = parse_program($code, "<test>");
+                    insta::assert_debug_snapshot!(parse_ast);
+                }
+            )*
+        }
+    }
+
+    function_and_lambda! {
+        test_function_no_args: "def f(): pass",
+        test_function_pos_args: "def f(a, b, c): pass",
+        test_function_pos_args_with_defaults: "def f(a, b=20, c=30): pass",
+        test_function_kw_only_args: "def f(*, a, b, c): pass",
+        test_function_kw_only_args_with_defaults: "def f(*, a, b=20, c=30): pass",
+        test_function_pos_and_kw_only_args: "def f(a, b, c, *, d, e, f): pass",
+        test_function_pos_and_kw_only_args_with_defaults: "def f(a, b, c, *, d, e=20, f=30): pass",
+        test_function_pos_and_kw_only_args_with_defaults_and_varargs: "def f(a, b, c, *args, d, e=20, f=30): pass",
+        test_function_pos_and_kw_only_args_with_defaults_and_varargs_and_kwargs: "def f(a, b, c, *args, d, e=20, f=30, **kwargs): pass",
+        test_lambda_no_args: "lambda: 1",
+        test_lambda_pos_args: "lambda a, b, c: 1",
+        test_lambda_pos_args_with_defaults: "lambda a, b=20, c=30: 1",
+        test_lambda_kw_only_args: "lambda *, a, b, c: 1",
+        test_lambda_kw_only_args_with_defaults: "lambda *, a, b=20, c=30: 1",
+        test_lambda_pos_and_kw_only_args: "lambda a, b, c, *, d, e: 0",
+    }
+
+    fn function_parse_error(src: &str) -> LexicalErrorType {
+        let parse_ast = parse_program(src, "<test>");
+        parse_ast
+            .map_err(|e| match e.error {
+                ParseErrorType::Lexical(e) => e,
+                _ => panic!("Expected LexicalError"),
+            })
+            .expect_err("Expected error")
+    }
+
+    macro_rules! function_and_lambda_error {
+        ($($name:ident: $code:expr, $error:expr,)*) => {
+            $(
+                #[test]
+                fn $name() {
+                    let error = function_parse_error($code);
+                    assert_eq!(error, $error);
+                }
+            )*
+        }
+    }
+
+    function_and_lambda_error! {
+        // Check definitions
+        test_duplicates_f1: "def f(a, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
+        test_duplicates_f2: "def f(a, *, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
+        test_duplicates_f3: "def f(a, a=20): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
+        test_duplicates_f4: "def f(a, *a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
+        test_duplicates_f5: "def f(a, *, **a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
+        test_duplicates_l1: "lambda a, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
+        test_duplicates_l2: "lambda a, *, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
+        test_duplicates_l3: "lambda a, a=20: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
+        test_duplicates_l4: "lambda a, *a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
+        test_duplicates_l5: "lambda a, *, **a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
+        test_default_arg_error_f: "def f(a, b=20, c): pass", LexicalErrorType::DefaultArgumentError,
+        test_default_arg_error_l: "lambda a, b=20, c: 1", LexicalErrorType::DefaultArgumentError,
+
+        // Check some calls.
+        test_positional_arg_error_f: "f(b=20, c)", LexicalErrorType::PositionalArgumentError,
+        test_unpacked_arg_error_f: "f(**b, *c)", LexicalErrorType::UnpackedArgumentError,
+        test_duplicate_kw_f1: "f(a=20, a=30)", LexicalErrorType::DuplicateKeywordArgumentError("a".to_string()),
+    }
+}
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args.snap b/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args.snap
new file mode 100644
index 00000000..bfb7074e
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args.snap
@@ -0,0 +1,108 @@
+---
+source: compiler/parser/src/function.rs
+assertion_line: 165
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 23,
+                },
+            ),
+            custom: (),
+            node: FunctionDef {
+                name: "f",
+                args: Arguments {
+                    posonlyargs: [],
+                    args: [],
+                    vararg: None,
+                    kwonlyargs: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 9,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "a",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "b",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 15,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 16,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "c",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                    kw_defaults: [],
+                    kwarg: None,
+                    defaults: [],
+                },
+                body: [
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 19,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 23,
+                            },
+                        ),
+                        custom: (),
+                        node: Pass,
+                    },
+                ],
+                decorator_list: [],
+                returns: None,
+                type_comment: None,
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args_with_defaults.snap b/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args_with_defaults.snap
new file mode 100644
index 00000000..2004bbee
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args_with_defaults.snap
@@ -0,0 +1,147 @@
+---
+source: compiler/parser/src/function.rs
+assertion_line: 165
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 29,
+                },
+            ),
+            custom: (),
+            node: FunctionDef {
+                name: "f",
+                args: Arguments {
+                    posonlyargs: [],
+                    args: [],
+                    vararg: None,
+                    kwonlyargs: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 9,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "a",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "b",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 18,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 19,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "c",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                    kw_defaults: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 14,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 16,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    20,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 20,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 22,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    30,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    kwarg: None,
+                    defaults: [],
+                },
+                body: [
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 25,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 29,
+                            },
+                        ),
+                        custom: (),
+                        node: Pass,
+                    },
+                ],
+                decorator_list: [],
+                returns: None,
+                type_comment: None,
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__function_no_args.snap b/parser/src/snapshots/rustpython_parser__function__tests__function_no_args.snap
new file mode 100644
index 00000000..a8d08b9a
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__function_no_args.snap
@@ -0,0 +1,52 @@
+---
+source: compiler/parser/src/function.rs
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 13,
+                },
+            ),
+            custom: (),
+            node: FunctionDef {
+                name: "f",
+                args: Arguments {
+                    posonlyargs: [],
+                    args: [],
+                    vararg: None,
+                    kwonlyargs: [],
+                    kw_defaults: [],
+                    kwarg: None,
+                    defaults: [],
+                },
+                body: [
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 13,
+                            },
+                        ),
+                        custom: (),
+                        node: Pass,
+                    },
+                ],
+                decorator_list: [],
+                returns: None,
+                type_comment: None,
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args.snap b/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args.snap
new file mode 100644
index 00000000..766a36b1
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args.snap
@@ -0,0 +1,162 @@
+---
+source: compiler/parser/src/function.rs
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 32,
+                },
+            ),
+            custom: (),
+            node: FunctionDef {
+                name: "f",
+                args: Arguments {
+                    posonlyargs: [],
+                    args: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 6,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 7,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "a",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 9,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "b",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "c",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                    vararg: None,
+                    kwonlyargs: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 18,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 19,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "d",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 21,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 22,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "e",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 24,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 25,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "f",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                    kw_defaults: [],
+                    kwarg: None,
+                    defaults: [],
+                },
+                body: [
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 28,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 32,
+                            },
+                        ),
+                        custom: (),
+                        node: Pass,
+                    },
+                ],
+                decorator_list: [],
+                returns: None,
+                type_comment: None,
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults.snap b/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults.snap
new file mode 100644
index 00000000..c6ff5e5d
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults.snap
@@ -0,0 +1,201 @@
+---
+source: compiler/parser/src/function.rs
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 38,
+                },
+            ),
+            custom: (),
+            node: FunctionDef {
+                name: "f",
+                args: Arguments {
+                    posonlyargs: [],
+                    args: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 6,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 7,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "a",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 9,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "b",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "c",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                    vararg: None,
+                    kwonlyargs: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 18,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 19,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "d",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 21,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 22,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "e",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 27,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 28,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "f",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                    kw_defaults: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 23,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 25,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    20,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 29,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 31,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    30,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    kwarg: None,
+                    defaults: [],
+                },
+                body: [
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 34,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 38,
+                            },
+                        ),
+                        custom: (),
+                        node: Pass,
+                    },
+                ],
+                decorator_list: [],
+                returns: None,
+                type_comment: None,
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs.snap b/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs.snap
new file mode 100644
index 00000000..830a9c52
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs.snap
@@ -0,0 +1,220 @@
+---
+source: compiler/parser/src/function.rs
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 42,
+                },
+            ),
+            custom: (),
+            node: FunctionDef {
+                name: "f",
+                args: Arguments {
+                    posonlyargs: [],
+                    args: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 6,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 7,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "a",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 9,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "b",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "c",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                    vararg: Some(
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 16,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 20,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "args",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ),
+                    kwonlyargs: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 22,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 23,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "d",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 25,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 26,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "e",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 31,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 32,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "f",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                    kw_defaults: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 27,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 29,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    20,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 33,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 35,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    30,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    kwarg: None,
+                    defaults: [],
+                },
+                body: [
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 38,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 42,
+                            },
+                        ),
+                        custom: (),
+                        node: Pass,
+                    },
+                ],
+                decorator_list: [],
+                returns: None,
+                type_comment: None,
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs_and_kwargs.snap b/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs_and_kwargs.snap
new file mode 100644
index 00000000..fde81e98
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__function_pos_and_kw_only_args_with_defaults_and_varargs_and_kwargs.snap
@@ -0,0 +1,239 @@
+---
+source: compiler/parser/src/function.rs
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 52,
+                },
+            ),
+            custom: (),
+            node: FunctionDef {
+                name: "f",
+                args: Arguments {
+                    posonlyargs: [],
+                    args: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 6,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 7,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "a",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 9,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "b",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "c",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                    vararg: Some(
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 16,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 20,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "args",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ),
+                    kwonlyargs: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 22,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 23,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "d",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 25,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 26,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "e",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 31,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 32,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "f",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                    kw_defaults: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 27,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 29,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    20,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 33,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 35,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    30,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    kwarg: Some(
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 39,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 45,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "kwargs",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ),
+                    defaults: [],
+                },
+                body: [
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 48,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 52,
+                            },
+                        ),
+                        custom: (),
+                        node: Pass,
+                    },
+                ],
+                decorator_list: [],
+                returns: None,
+                type_comment: None,
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__function_pos_args.snap b/parser/src/snapshots/rustpython_parser__function__tests__function_pos_args.snap
new file mode 100644
index 00000000..251bba87
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__function_pos_args.snap
@@ -0,0 +1,107 @@
+---
+source: compiler/parser/src/function.rs
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 20,
+                },
+            ),
+            custom: (),
+            node: FunctionDef {
+                name: "f",
+                args: Arguments {
+                    posonlyargs: [],
+                    args: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 6,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 7,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "a",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 9,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "b",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 12,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "c",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                    vararg: None,
+                    kwonlyargs: [],
+                    kw_defaults: [],
+                    kwarg: None,
+                    defaults: [],
+                },
+                body: [
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 16,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 20,
+                            },
+                        ),
+                        custom: (),
+                        node: Pass,
+                    },
+                ],
+                decorator_list: [],
+                returns: None,
+                type_comment: None,
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__function_pos_args_with_defaults.snap b/parser/src/snapshots/rustpython_parser__function__tests__function_pos_args_with_defaults.snap
new file mode 100644
index 00000000..71b4bb86
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__function_pos_args_with_defaults.snap
@@ -0,0 +1,146 @@
+---
+source: compiler/parser/src/function.rs
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 26,
+                },
+            ),
+            custom: (),
+            node: FunctionDef {
+                name: "f",
+                args: Arguments {
+                    posonlyargs: [],
+                    args: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 6,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 7,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "a",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 9,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 10,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "b",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 15,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 16,
+                                },
+                            ),
+                            custom: (),
+                            node: ArgData {
+                                arg: "c",
+                                annotation: None,
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                    vararg: None,
+                    kwonlyargs: [],
+                    kw_defaults: [],
+                    kwarg: None,
+                    defaults: [
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 11,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    20,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 1,
+                                column: 17,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 19,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    30,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                },
+                body: [
+                    Located {
+                        location: Location {
+                            row: 1,
+                            column: 22,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 26,
+                            },
+                        ),
+                        custom: (),
+                        node: Pass,
+                    },
+                ],
+                decorator_list: [],
+                returns: None,
+                type_comment: None,
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args.snap b/parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args.snap
new file mode 100644
index 00000000..59987372
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args.snap
@@ -0,0 +1,121 @@
+---
+source: compiler/parser/src/function.rs
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 20,
+                },
+            ),
+            custom: (),
+            node: Expr {
+                value: Located {
+                    location: Location {
+                        row: 1,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 20,
+                        },
+                    ),
+                    custom: (),
+                    node: Lambda {
+                        args: Arguments {
+                            posonlyargs: [],
+                            args: [],
+                            vararg: None,
+                            kwonlyargs: [
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "a",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "b",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 16,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 17,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "c",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                            ],
+                            kw_defaults: [],
+                            kwarg: None,
+                            defaults: [],
+                        },
+                        body: Located {
+                            location: Location {
+                                row: 1,
+                                column: 19,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 20,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args_with_defaults.snap b/parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args_with_defaults.snap
new file mode 100644
index 00000000..d48792dc
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__lambda_kw_only_args_with_defaults.snap
@@ -0,0 +1,160 @@
+---
+source: compiler/parser/src/function.rs
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 26,
+                },
+            ),
+            custom: (),
+            node: Expr {
+                value: Located {
+                    location: Location {
+                        row: 1,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 26,
+                        },
+                    ),
+                    custom: (),
+                    node: Lambda {
+                        args: Arguments {
+                            posonlyargs: [],
+                            args: [],
+                            vararg: None,
+                            kwonlyargs: [
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "a",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "b",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 19,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 20,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "c",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                            ],
+                            kw_defaults: [
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 15,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 17,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            20,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 21,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 23,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            30,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                            kwarg: None,
+                            defaults: [],
+                        },
+                        body: Located {
+                            location: Location {
+                                row: 1,
+                                column: 25,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 26,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__lambda_no_args.snap b/parser/src/snapshots/rustpython_parser__function__tests__lambda_no_args.snap
new file mode 100644
index 00000000..aab5ec4d
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__lambda_no_args.snap
@@ -0,0 +1,66 @@
+---
+source: compiler/parser/src/function.rs
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 9,
+                },
+            ),
+            custom: (),
+            node: Expr {
+                value: Located {
+                    location: Location {
+                        row: 1,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 9,
+                        },
+                    ),
+                    custom: (),
+                    node: Lambda {
+                        args: Arguments {
+                            posonlyargs: [],
+                            args: [],
+                            vararg: None,
+                            kwonlyargs: [],
+                            kw_defaults: [],
+                            kwarg: None,
+                            defaults: [],
+                        },
+                        body: Located {
+                            location: Location {
+                                row: 1,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 9,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_and_kw_only_args.snap b/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_and_kw_only_args.snap
new file mode 100644
index 00000000..22f695b3
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_and_kw_only_args.snap
@@ -0,0 +1,158 @@
+---
+source: compiler/parser/src/function.rs
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 26,
+                },
+            ),
+            custom: (),
+            node: Expr {
+                value: Located {
+                    location: Location {
+                        row: 1,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 26,
+                        },
+                    ),
+                    custom: (),
+                    node: Lambda {
+                        args: Arguments {
+                            posonlyargs: [],
+                            args: [
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 7,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 8,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "a",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "b",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "c",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                            ],
+                            vararg: None,
+                            kwonlyargs: [
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 19,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 20,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "d",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 22,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 23,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "e",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                            ],
+                            kw_defaults: [],
+                            kwarg: None,
+                            defaults: [],
+                        },
+                        body: Located {
+                            location: Location {
+                                row: 1,
+                                column: 25,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 26,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    0,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args.snap b/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args.snap
new file mode 100644
index 00000000..f4ce79f9
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args.snap
@@ -0,0 +1,121 @@
+---
+source: compiler/parser/src/function.rs
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 17,
+                },
+            ),
+            custom: (),
+            node: Expr {
+                value: Located {
+                    location: Location {
+                        row: 1,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 17,
+                        },
+                    ),
+                    custom: (),
+                    node: Lambda {
+                        args: Arguments {
+                            posonlyargs: [],
+                            args: [
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 7,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 8,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "a",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "b",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "c",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                            ],
+                            vararg: None,
+                            kwonlyargs: [],
+                            kw_defaults: [],
+                            kwarg: None,
+                            defaults: [],
+                        },
+                        body: Located {
+                            location: Location {
+                                row: 1,
+                                column: 16,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 17,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    ],
+)
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args_with_defaults.snap b/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args_with_defaults.snap
new file mode 100644
index 00000000..01691aa7
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__function__tests__lambda_pos_args_with_defaults.snap
@@ -0,0 +1,160 @@
+---
+source: compiler/parser/src/function.rs
+expression: parse_ast
+---
+Ok(
+    [
+        Located {
+            location: Location {
+                row: 1,
+                column: 0,
+            },
+            end_location: Some(
+                Location {
+                    row: 1,
+                    column: 23,
+                },
+            ),
+            custom: (),
+            node: Expr {
+                value: Located {
+                    location: Location {
+                        row: 1,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 23,
+                        },
+                    ),
+                    custom: (),
+                    node: Lambda {
+                        args: Arguments {
+                            posonlyargs: [],
+                            args: [
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 7,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 8,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "a",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "b",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 16,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 17,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: ArgData {
+                                        arg: "c",
+                                        annotation: None,
+                                        type_comment: None,
+                                    },
+                                },
+                            ],
+                            vararg: None,
+                            kwonlyargs: [],
+                            kw_defaults: [],
+                            kwarg: None,
+                            defaults: [
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            20,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 18,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 20,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            30,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                        },
+                        body: Located {
+                            location: Location {
+                                row: 1,
+                                column: 22,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 1,
+                                    column: 23,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    ],
+)

From 66cbe3fe62186512d142705624f7a99be3c3bc9d Mon Sep 17 00:00:00 2001
From: Dimitris Fasarakis Hilliard <d.f.hilliard@gmail.com>
Date: Tue, 7 Feb 2023 21:42:15 +0200
Subject: [PATCH 416/459] Document parser crate.

---
 parser/src/context.rs |   2 +-
 parser/src/error.rs   |  66 +++++++++++++--
 parser/src/lib.rs     | 118 ++++++++++++++++++++++++--
 parser/src/mode.rs    |   6 ++
 parser/src/parser.rs  | 187 +++++++++++++++++++++++++++++++-----------
 parser/src/token.rs   | 118 +++++++++++++++++++++++---
 6 files changed, 420 insertions(+), 77 deletions(-)

diff --git a/parser/src/context.rs b/parser/src/context.rs
index f10e1054..eb1953f6 100644
--- a/parser/src/context.rs
+++ b/parser/src/context.rs
@@ -1,6 +1,6 @@
 use rustpython_ast::{Expr, ExprContext, ExprKind};
 
-pub fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
+pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
     match expr.node {
         ExprKind::Name { id, .. } => Expr {
             node: ExprKind::Name { id, ctx },
diff --git a/parser/src/error.rs b/parser/src/error.rs
index 4edc02ea..dab07f8c 100644
--- a/parser/src/error.rs
+++ b/parser/src/error.rs
@@ -1,40 +1,71 @@
-//! Define internal parse error types
-//! The goal is to provide a matching and a safe error API, maksing errors from LALR
+//! Error types for the parser.
+//!
+//! These types are used to represent errors that occur during lexing and parsing and are
+//! returned by the `parse_*` functions in the [parser] module and the iterator in the
+//! [lexer] implementation.
+//!
+//! [parser]: crate::parser
+//! [lexer]: crate::lexer
 
+// Define internal parse error types.
+// The goal is to provide a matching and a safe error API, masking errors from LALR
 use crate::{ast::Location, token::Tok};
 use lalrpop_util::ParseError as LalrpopError;
 use std::fmt;
 
-/// Represents an error during lexical scanning.
+/// Represents an error during lexing.
 #[derive(Debug, PartialEq)]
 pub struct LexicalError {
+    /// The type of error that occurred.
     pub error: LexicalErrorType,
+    /// The location of the error.
     pub location: Location,
 }
 
 impl LexicalError {
+    /// Creates a new `LexicalError` with the given error type and location.
     pub fn new(error: LexicalErrorType, location: Location) -> Self {
         Self { error, location }
     }
 }
 
+/// Represents the different types of errors that can occur during lexing.
 #[derive(Debug, PartialEq)]
 pub enum LexicalErrorType {
+    // TODO: Can probably be removed, the places it is used seem to be able
+    // to use the `UnicodeError` variant instead.
+    #[doc(hidden)]
     StringError,
+    // TODO: Should take a start/end position to report.
+    /// Decoding of a unicode escape sequence in a string literal failed.
     UnicodeError,
+    /// The nesting of brackets/braces/parentheses is not balanced.
     NestingError,
+    /// The indentation is not consistent.
     IndentationError,
+    /// Inconsistent use of tabs and spaces.
     TabError,
+    /// Encountered a tab after a space.
     TabsAfterSpaces,
+    /// A non-default argument follows a default argument.
     DefaultArgumentError,
+    /// A duplicate argument was found in a function definition.
     DuplicateArgumentError(String),
+    /// A positional argument follows a keyword argument.
     PositionalArgumentError,
+    /// An iterable argument unpacking `*args` follows keyword argument unpacking `**kwargs`.
     UnpackedArgumentError,
+    /// A keyword argument was repeated.
     DuplicateKeywordArgumentError(String),
+    /// An unrecognized token was encountered.
     UnrecognizedToken { tok: char },
+    /// An f-string error containing the [`FStringErrorType`].
     FStringError(FStringErrorType),
+    /// An unexpected character was encountered after a line continuation.
     LineContinuationError,
+    /// An unexpected end of file was encountered.
     Eof,
+    /// An unexpected error occurred.
     OtherError(String),
 }
 
@@ -85,13 +116,17 @@ impl fmt::Display for LexicalErrorType {
 }
 
 // TODO: consolidate these with ParseError
+/// An error that occurred during parsing of an f-string.
 #[derive(Debug, PartialEq)]
 pub struct FStringError {
+    /// The type of error that occurred.
     pub error: FStringErrorType,
+    /// The location of the error.
     pub location: Location,
 }
 
 impl FStringError {
+    /// Creates a new `FStringError` with the given error type and location.
     pub fn new(error: FStringErrorType, location: Location) -> Self {
         Self { error, location }
     }
@@ -106,19 +141,33 @@ impl From<FStringError> for LexicalError {
     }
 }
 
+/// Represents the different types of errors that can occur during parsing of an f-string.
 #[derive(Debug, PartialEq)]
 pub enum FStringErrorType {
+    /// Expected a right brace after an opened left brace.
     UnclosedLbrace,
+    /// Expected a left brace after an ending right brace.
     UnopenedRbrace,
+    /// Expected a right brace after a conversion flag.
     ExpectedRbrace,
+    /// An error occurred while parsing an f-string expression.
     InvalidExpression(Box<ParseErrorType>),
+    /// An invalid conversion flag was encountered.
     InvalidConversionFlag,
+    /// An empty expression was encountered.
     EmptyExpression,
+    /// An opening delimiter was not closed properly.
     MismatchedDelimiter(char, char),
+    /// Too many nested expressions in an f-string.
     ExpressionNestedTooDeeply,
+    /// The f-string expression cannot include the given character.
     ExpressionCannotInclude(char),
+    /// A single right brace was encountered.
     SingleRbrace,
+    /// A closing delimiter was not opened properly.
     Unmatched(char),
+    // TODO: Test this case.
+    /// Unterminated string.
     UnterminatedString,
 }
 
@@ -167,9 +216,10 @@ impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
     }
 }
 
-/// Represents an error during parsing
+/// Represents an error during parsing.
 pub type ParseError = rustpython_compiler_core::BaseError<ParseErrorType>;
 
+/// Represents the different types of errors that can occur during parsing.
 #[derive(Debug, PartialEq, thiserror::Error)]
 pub enum ParseErrorType {
     /// Parser encountered an unexpected end of input
@@ -180,11 +230,12 @@ pub enum ParseErrorType {
     InvalidToken,
     /// Parser encountered an unexpected token
     UnrecognizedToken(Tok, Option<String>),
-    /// Maps to `User` type from `lalrpop-util`
+    // Maps to `User` type from `lalrpop-util`
+    /// Parser encountered an error during lexing.
     Lexical(LexicalErrorType),
 }
 
-/// Convert `lalrpop_util::ParseError` to our internal type
+// Convert `lalrpop_util::ParseError` to our internal type
 pub(crate) fn parse_error_from_lalrpop(
     err: LalrpopError<Location, Tok, LexicalError>,
     source_path: &str,
@@ -258,6 +309,7 @@ impl fmt::Display for ParseErrorType {
 }
 
 impl ParseErrorType {
+    /// Returns true if the error is an indentation error.
     pub fn is_indentation_error(&self) -> bool {
         match self {
             ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true,
@@ -267,6 +319,8 @@ impl ParseErrorType {
             _ => false,
         }
     }
+
+    /// Returns true if the error is a tab error.
     pub fn is_tab_error(&self) -> bool {
         matches!(
             self,
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index ce9dde1d..e258acf9 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -1,19 +1,119 @@
-//! This crate can be used to parse python sourcecode into a so
-//! called AST (abstract syntax tree).
+//! This crate can be used to parse Python source code into an Abstract
+//! Syntax Tree.
 //!
-//! The stages involved in this process are lexical analysis and
-//! parsing. The lexical analysis splits the sourcecode into
-//! tokens, and the parsing transforms those tokens into an AST.
+//! ## Overview:
 //!
-//! For example, one could do this:
+//! The process by which source code is parsed into an AST can be broken down
+//! into two general stages: [lexical analysis] and [parsing].
 //!
+//! During lexical analysis, the source code is converted into a stream of lexical
+//! tokens that represent the smallest meaningful units of the language. For example,
+//! the source code `print("Hello world")` would _roughly_ be converted into the following
+//! stream of tokens:
+//!
+//! ```text
+//! Name("print"), LeftParen, String("Hello world"), RightParen
 //! ```
-//! use rustpython_parser::{parser, ast};
 //!
-//! let python_source = "print('Hello world')";
-//! let python_ast = parser::parse_expression(python_source, "<embedded>").unwrap();
+//! these tokens are then consumed by the parser, which matches them against a set of
+//! grammar rules to verify that the source code is syntactically valid and to construct
+//! an AST that represents the source code.
+//!  
+//! During parsing, the parser consumes the tokens generated by the lexer and constructs
+//! a tree representation of the source code. The tree is made up of nodes that represent
+//! the different syntactic constructs of the language. If the source code is syntactically
+//! invalid, parsing fails and an error is returned. After a successful parse, the AST can
+//! be used to perform further analysis on the source code. Continuing with the example
+//! above, the AST generated by the parser would _roughly_ look something like this:
+//!
+//! ```text
+//! node: Expr {
+//!     value: {
+//!         node: Call {
+//!             func: {
+//!                 node: Name {
+//!                     id: "print",
+//!                     ctx: Load,
+//!                 },
+//!             },
+//!             args: [
+//!                 node: Constant {
+//!                     value: Str("Hello World"),
+//!                     kind: None,
+//!                 },
+//!             ],
+//!             keywords: [],
+//!         },
+//!     },
+//! },
+//!```
+//!
+//! Note: The Tokens/ASTs shown above are not the exact tokens/ASTs generated by the parser.
+//!
+//! ## Source code layout:
+//!
+//! The functionality of this crate is split into several modules:
+//!
+//! - [token]: This module contains the definition of the tokens that are generated by the lexer.
+//! - [lexer]: This module contains the lexer and is responsible for generating the tokens.
+//! - [parser]: This module contains an interface to the parser and is responsible for generating the AST.
+//!     - Functions and strings have special parsing requirements that are handled in additional files.
+//! - [mode]: This module contains the definition of the different modes that the parser can be in.
+//! - [error]: This module contains the definition of the errors that can be returned by the parser.
 //!
+//! # Examples
+//!
+//! For example, to get a stream of tokens from a given string, one could do this:
+//!
+//! ```
+//! use rustpython_parser::lexer::make_tokenizer;
+//!
+//! let python_source = r#"
+//! def is_odd(i):
+//!     return bool(i & 1)
+//! "#;
+//! let mut tokens = make_tokenizer(python_source);
+//! assert!(tokens.all(|t| t.is_ok()));
 //! ```
+//!
+//! These tokens can be directly fed into the parser to generate an AST:
+//!
+//! ```
+//! use rustpython_parser::parser::{parse_tokens, Mode};
+//! use rustpython_parser::lexer::make_tokenizer;
+//!
+//! let python_source = r#"
+//! def is_odd(i):
+//!    return bool(i & 1)
+//! "#;
+//! let tokens = make_tokenizer(python_source);
+//! let ast = parse_tokens(tokens, Mode::Module, "<embedded>");
+//!
+//! assert!(ast.is_ok());
+//! ```
+//!
+//! Alternatively, you can use one of the other `parse_*` functions to parse a string directly without using a specific
+//! mode or tokenizing the source beforehand:
+//!
+//! ```
+//! use rustpython_parser::parser::parse_program;
+//!
+//! let python_source = r#"
+//! def is_odd(i):
+//!   return bool(i & 1)
+//! "#;
+//! let ast = parse_program(python_source, "<embedded>");
+//!
+//! assert!(ast.is_ok());
+//! ```
+//!
+//! [lexical analysis]: https://en.wikipedia.org/wiki/Lexical_analysis
+//! [parsing]: https://en.wikipedia.org/wiki/Parsing
+//! [token]: crate::token
+//! [lexer]: crate::lexer
+//! [parser]: crate::parser
+//! [mode]: crate::mode
+//! [error]: crate::error
 
 #![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
 #![doc(html_root_url = "https://docs.rs/rustpython-parser/")]
diff --git a/parser/src/mode.rs b/parser/src/mode.rs
index cd84a098..4403fbee 100644
--- a/parser/src/mode.rs
+++ b/parser/src/mode.rs
@@ -1,9 +1,14 @@
+//! Control in the different modes by which a source file can be parsed.
 use crate::token::Tok;
 
+/// The mode argument specifies in what way code must be parsed.
 #[derive(Clone, Copy)]
 pub enum Mode {
+    /// The code consists of a sequence of statements.
     Module,
+    /// The code consists of a sequence of interactive statement.
     Interactive,
+    /// The code consists of a single expression.
     Expression,
 }
 
@@ -39,6 +44,7 @@ impl std::str::FromStr for Mode {
     }
 }
 
+/// Returned when a given mode is not valid.
 #[derive(Debug)]
 pub struct ModeParseError {
     _priv: (),
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 38b065a5..7f828021 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -1,9 +1,16 @@
-//! Python parsing.
+//! Contains the interface to the Python parser.
 //!
-//! Use this module to parse python code into an AST.
-//! There are three ways to parse python code. You could
-//! parse a whole program, a single statement, or a single
-//! expression.
+//! Functions in this module can be used to parse Python code into an [Abstract Syntax Tree]
+//! (AST) that is then transformed into bytecode.
+//!
+//! There are three ways to parse Python code corresponding to the different [`Mode`]s
+//! defined in the [`mode`] module.
+//!
+//! All functions return a [`Result`](std::result::Result) containing the parsed AST or
+//! a [`ParseError`] if parsing failed.
+//!
+//! [Abstract Syntax Tree]: https://en.wikipedia.org/wiki/Abstract_syntax_tree
+//! [`Mode`]: crate::mode
 
 use crate::lexer::{LexResult, Tok};
 pub use crate::mode::Mode;
@@ -12,13 +19,26 @@ use ast::Location;
 use itertools::Itertools;
 use std::iter;
 
-/*
- * Parse python code.
- * Grammar may be inspired by antlr grammar for python:
- * https://github.com/antlr/grammars-v4/tree/master/python3
- */
-
-/// Parse a full python program, containing usually multiple lines.
+/// Parse a full Python program usually consisting of multiple lines.
+///  
+/// This is a convenience function that can be used to parse a full Python program without having to
+/// specify the [`Mode`] or the location. It is probably what you want to use most of the time.
+///
+/// # Example
+///
+/// For example, parsing a simple function definition and a call to that function:
+///
+/// ```
+/// use rustpython_parser::parser;
+/// let source = r#"
+/// def foo():
+///    return 42
+///
+/// print(foo())
+/// "#;
+/// let program = parser::parse_program(source, "<embedded>");
+/// assert!(program.is_ok());
+/// ```
 pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, ParseError> {
     parse(source, Mode::Module, source_path).map(|top| match top {
         ast::Mod::Module { body, .. } => body,
@@ -26,49 +46,44 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
     })
 }
 
-/// Parses a python expression
+/// Parses a single Python expression.
+///
+/// This convenience function can be used to parse a single expression without having to
+/// specify the Mode or the location.
 ///
 /// # Example
-/// ```
+///
+/// For example, parsing a single expression denoting the addition of two numbers:
+///
+///  ```
 /// extern crate num_bigint;
 /// use rustpython_parser::{parser, ast};
-/// let expr = parser::parse_expression("1 + 2", "<embedded>").unwrap();
-///
-/// assert_eq!(
-///     expr,
-///     ast::Expr {
-///         location: ast::Location::new(1, 0),
-///         end_location: Some(ast::Location::new(1, 5)),
-///         custom: (),
-///         node: ast::ExprKind::BinOp {
-///             left: Box::new(ast::Expr {
-///                 location: ast::Location::new(1, 0),
-///                 end_location: Some(ast::Location::new(1, 1)),
-///                 custom: (),
-///                 node: ast::ExprKind::Constant {
-///                     value: ast::Constant::Int(1.into()),
-///                     kind: None,
-///                 }
-///             }),
-///             op: ast::Operator::Add,
-///             right: Box::new(ast::Expr {
-///                 location: ast::Location::new(1, 4),
-///                 end_location: Some(ast::Location::new(1, 5)),
-///                 custom: (),
-///                 node: ast::ExprKind::Constant {
-///                     value: ast::Constant::Int(2.into()),
-///                     kind: None,
-///                 }
-///             })
-///         }
-///     },
-/// );
+/// let expr = parser::parse_expression("1 + 2", "<embedded>");
+///
+/// assert!(expr.is_ok());
 ///
 /// ```
 pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> {
     parse_expression_located(source, path, Location::new(1, 0))
 }
 
+/// Parses a Python expression from a given location.
+///
+/// This function allows to specify the location of the expression in the source code, other than
+/// that, it behaves exactly like [`parse_expression`].
+///
+/// # Example
+///
+/// Parsing a single expression denoting the addition of two numbers, but this time specifying a different,
+/// somewhat silly, location:
+///
+/// ```
+/// use rustpython_parser::parser::parse_expression_located;
+/// use rustpython_parser::ast::Location;
+///
+/// let expr = parse_expression_located("1 + 2", "<embedded>", Location::new(5, 20));
+/// assert!(expr.is_ok());
+/// ```
 pub fn parse_expression_located(
     source: &str,
     path: &str,
@@ -80,12 +95,64 @@ pub fn parse_expression_located(
     })
 }
 
-// Parse a given source code
+/// Parse the given Python source code using the specified [`Mode`].
+///
+/// This function is the most general function to parse Python code. Based on the [`Mode`] supplied,
+/// it can be used to parse a single expression, a full Python program or an interactive expression.
+///
+/// # Example
+///
+/// If we want to parse a simple expression, we can use the [`Mode::Expression`] mode during
+/// parsing:
+///
+/// ```
+/// use rustpython_parser::parser::{parse, Mode};
+///
+/// let expr = parse("1 + 2", Mode::Expression, "<embedded>");
+/// assert!(expr.is_ok());
+/// ```
+///
+/// Alternatively, we can parse a full Python program consisting of multiple lines:
+///
+/// ```
+/// use rustpython_parser::parser::{parse, Mode};
+///
+/// let source = r#"
+/// class Greeter:
+///
+///   def greet(self):
+///    print("Hello, world!")
+/// "#;
+/// let program = parse(source, Mode::Module, "<embedded>");
+/// assert!(program.is_ok());
+/// ```
 pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, ParseError> {
     parse_located(source, mode, source_path, Location::new(1, 0))
 }
 
-// Parse a given source code from a given location
+/// Parse the given Python source code using the specified [`Mode`] and [`Location`].
+///
+/// This function allows to specify the location of the the source code, other than
+/// that, it behaves exactly like [`parse`].
+///
+/// # Example
+///
+/// ```
+/// use rustpython_parser::parser::{parse_located, Mode};
+/// use rustpython_parser::ast::Location;
+///
+/// let source = r#"
+/// def fib(i):
+///    a, b = 0, 1
+///    for _ in range(i):
+///       a, b = b, a + b
+///    return a
+///
+/// print(fib(42))
+/// "#;
+/// let program = parse_located(source, Mode::Module, "<embedded>", Location::new(1, 0));
+/// assert!(program.is_ok());
+/// ```
 pub fn parse_located(
     source: &str,
     mode: Mode,
@@ -96,7 +163,22 @@ pub fn parse_located(
     parse_tokens(lxr, mode, source_path)
 }
 
-// Parse a given token iterator.
+/// Parse an iterator of [`LexResult`]s using the specified [`Mode`].
+///
+/// This could allow you to perform some preprocessing on the tokens before parsing them.
+///
+/// # Example
+///
+/// As an example, instead of parsing a string, we can parse a list of tokens after we generate
+/// them using the [`lexer::make_tokenizer`] function:
+///
+/// ```
+/// use rustpython_parser::parser::{parse_tokens, Mode};
+/// use rustpython_parser::lexer::make_tokenizer;
+///
+/// let expr = parse_tokens(make_tokenizer("1 + 2"), Mode::Expression, "<embedded>");
+/// assert!(expr.is_ok());
+/// ```
 pub fn parse_tokens(
     lxr: impl IntoIterator<Item = LexResult>,
     mode: Mode,
@@ -328,4 +410,13 @@ with (0 as a, 1 as b,): pass
         let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
+
+    #[test]
+    fn test_modes() {
+        let source = "a[0][1][2][3][4]";
+
+        assert!(parse(&source, Mode::Expression, "<embedded>").is_ok());
+        assert!(parse(&source, Mode::Module, "<embedded>").is_ok());
+        assert!(parse(&source, Mode::Interactive, "<embedded>").is_ok());
+    }
 }
diff --git a/parser/src/token.rs b/parser/src/token.rs
index b51b2f47..3fa6ff14 100644
--- a/parser/src/token.rs
+++ b/parser/src/token.rs
@@ -1,86 +1,154 @@
-//! Different token definitions.
-//! Loosely based on token.h from CPython source:
+//! Token type for Python source code created by the lexer and consumed by the parser.
+//!
+//! This module defines the tokens that the lexer recognizes. The tokens are
+//! loosely based on the token definitions found in the [CPython source].
+//!
+//! [CPython source]: https://github.com/python/cpython/blob/dfc2e065a2e71011017077e549cd2f9bf4944c54/Include/internal/pycore_token.h
 use num_bigint::BigInt;
 use std::fmt;
 
-/// Python source code can be tokenized in a sequence of these tokens.
+/// The set of tokens the Python source code can be tokenized in.
 #[derive(Clone, Debug, PartialEq)]
 pub enum Tok {
+    /// Token value for a name, commonly known as an identifier.
     Name {
+        /// The name value.
         name: String,
     },
+    /// Token value for an integer.
     Int {
+        /// The integer value.
         value: BigInt,
     },
+    /// Token value for a floating point number.
     Float {
+        /// The float value.
         value: f64,
     },
+    /// Token value for a complex number.
     Complex {
+        /// The real part of the complex number.
         real: f64,
+        /// The imaginary part of the complex number.
         imag: f64,
     },
+    /// Token value for a string.
     String {
+        /// The string value.
         value: String,
+        /// The kind of string.
         kind: StringKind,
+        /// Whether the string is triple quoted.
         triple_quoted: bool,
     },
+    /// Token value for a comment. These are filtered out of the token stream prior to parsing.
+    Comment(String),
+    /// Token value for a newline.
     Newline,
+    /// Token value for a newline that is not a logical line break. These are filtered out of
+    /// the token stream prior to parsing.
     NonLogicalNewline,
+    /// Token value for an indent.
     Indent,
+    /// Token value for a dedent.
     Dedent,
-    StartModule,
-    StartInteractive,
-    StartExpression,
     EndOfFile,
+    /// Token value for a left parenthesis `(`.
     Lpar,
+    /// Token value for a right parenthesis `)`.
     Rpar,
+    /// Token value for a left square bracket `[`.
     Lsqb,
+    /// Token value for a right square bracket `]`.
     Rsqb,
+    /// Token value for a colon `:`.
     Colon,
+    /// Token value for a comma `,`.
     Comma,
-    Comment(String),
+    /// Token value for a semicolon `;`.
     Semi,
+    /// Token value for plus `+`.
     Plus,
+    /// Token value for minus `-`.
     Minus,
+    /// Token value for star `*`.
     Star,
+    /// Token value for slash `/`.
     Slash,
-    Vbar,  // '|'
-    Amper, // '&'
+    /// Token value for vertical bar `|`.
+    Vbar,
+    /// Token value for ampersand `&`.
+    Amper,
+    /// Token value for less than `<`.
     Less,
+    /// Token value for greater than `>`.
     Greater,
+    /// Token value for equal `=`.
     Equal,
+    /// Token value for dot `.`.
     Dot,
+    /// Token value for percent `%`.
     Percent,
+    /// Token value for left bracket `{`.
     Lbrace,
+    /// Token value for right bracket `}`.
     Rbrace,
+    /// Token value for double equal `==`.
     EqEqual,
+    /// Token value for not equal `!=`.
     NotEqual,
+    /// Token value for less than or equal `<=`.
     LessEqual,
+    /// Token value for greater than or equal `>=`.
     GreaterEqual,
+    /// Token value for tilde `~`.
     Tilde,
+    /// Token value for caret `^`.
     CircumFlex,
+    /// Token value for left shift `<<`.
     LeftShift,
+    /// Token value for right shift `>>`.
     RightShift,
+    /// Token value for double star `**`.
     DoubleStar,
-    DoubleStarEqual, // '**='
+    /// Token value for double star equal `**=`.
+    DoubleStarEqual,
+    /// Token value for plus equal `+=`.
     PlusEqual,
+    /// Token value for minus equal `-=`.
     MinusEqual,
+    /// Token value for star equal `*=`.
     StarEqual,
+    /// Token value for slash equal `/=`.
     SlashEqual,
+    /// Token value for percent equal `%=`.
     PercentEqual,
-    AmperEqual, // '&='
+    /// Token value for ampersand equal `&=`.
+    AmperEqual,
+    /// Token value for vertical bar equal `|=`.
     VbarEqual,
-    CircumflexEqual, // '^='
+    /// Token value for caret equal `^=`.
+    CircumflexEqual,
+    /// Token value for left shift equal `<<=`.
     LeftShiftEqual,
+    /// Token value for right shift equal `>>=`.
     RightShiftEqual,
-    DoubleSlash, // '//'
+    /// Token value for double slash `//`.
+    DoubleSlash,
+    /// Token value for double slash equal `//=`.
     DoubleSlashEqual,
+    /// Token value for colon equal `:=`.
     ColonEqual,
+    /// Token value for at `@`.
     At,
+    /// Token value for at equal `@=`.
     AtEqual,
+    /// Token value for arrow `->`.
     Rarrow,
+    /// Token value for ellipsis `...`.
     Ellipsis,
 
+    // Self documenting.
     // Keywords (alphabetically):
     False,
     None,
@@ -118,6 +186,11 @@ pub enum Tok {
     While,
     With,
     Yield,
+
+    // RustPython specific.
+    StartModule,
+    StartInteractive,
+    StartExpression,
 }
 
 impl fmt::Display for Tok {
@@ -231,14 +304,25 @@ impl fmt::Display for Tok {
     }
 }
 
+/// The kind of string literal as described in the [String and Bytes literals]
+/// section of the Python reference.
+///
+/// [String and Bytes literals]: https://docs.python.org/3/reference/lexical_analysis.html#string-and-bytes-literals
 #[derive(PartialEq, Eq, Debug, Clone)]
 pub enum StringKind {
+    /// A normal string literal with no prefix.
     String,
+    /// A f-string literal, with a `f` or `F` prefix.
     FString,
+    /// A byte string literal, with a `b` or `B` prefix.
     Bytes,
+    /// A raw string literal, with a `r` or `R` prefix.
     RawString,
+    /// A raw f-string literal, with a `rf`/`fr` or `rF`/`Fr` or `Rf`/`fR` or `RF`/`FR` prefix.
     RawFString,
+    /// A raw byte string literal, with a `rb`/`br` or `rB`/`Br` or `Rb`/`bR` or `RB`/`BR` prefix.
     RawBytes,
+    /// A unicode string literal, with a `u` or `U` prefix.
     Unicode,
 }
 
@@ -286,25 +370,33 @@ impl fmt::Display for StringKind {
 }
 
 impl StringKind {
+    /// Returns true if the string is a raw string, i,e one of
+    /// [`StringKind::RawString`] or [`StringKind::RawFString`] or [`StringKind::RawBytes`].
     pub fn is_raw(&self) -> bool {
         use StringKind::{RawBytes, RawFString, RawString};
         matches!(self, RawString | RawFString | RawBytes)
     }
 
+    /// Returns true if the string is an f-string, i,e one of
+    /// [`StringKind::FString`] or [`StringKind::RawFString`].
     pub fn is_fstring(&self) -> bool {
         use StringKind::{FString, RawFString};
         matches!(self, FString | RawFString)
     }
 
+    /// Returns true if the string is a byte string, i,e one of
+    /// [`StringKind::Bytes`] or [`StringKind::RawBytes`].
     pub fn is_bytes(&self) -> bool {
         use StringKind::{Bytes, RawBytes};
         matches!(self, Bytes | RawBytes)
     }
 
+    /// Returns true if the string is a unicode string, i,e [`StringKind::Unicode`].
     pub fn is_unicode(&self) -> bool {
         matches!(self, StringKind::Unicode)
     }
 
+    /// Returns the number of characters in the prefix.
     pub fn prefix_len(&self) -> usize {
         use StringKind::*;
         match self {

From 72ddb53d0f4be73af4c60bbf7167c75970905f6b Mon Sep 17 00:00:00 2001
From: John Pham <minh.pham2000@linecorp.com>
Date: Sun, 19 Feb 2023 20:09:54 +0700
Subject: [PATCH 417/459] Make common::repr throw error instead of panic
 (#4520)

---
 ast/src/constant.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ast/src/constant.rs b/ast/src/constant.rs
index adec47a1..54aa7594 100644
--- a/ast/src/constant.rs
+++ b/ast/src/constant.rs
@@ -1,3 +1,5 @@
+use std::fmt::Error;
+
 use num_bigint::BigInt;
 pub use rustpython_compiler_core::ConversionFlag;
 
@@ -42,7 +44,7 @@ impl std::fmt::Display for Constant {
             Constant::None => f.pad("None"),
             Constant::Bool(b) => f.pad(if *b { "True" } else { "False" }),
             Constant::Str(s) => rustpython_common::str::repr(s).fmt(f),
-            Constant::Bytes(b) => f.pad(&rustpython_common::bytes::repr(b)),
+            Constant::Bytes(b) => f.pad(&rustpython_common::bytes::repr(b).map_err(|_err| Error)?),
             Constant::Int(i) => i.fmt(f),
             Constant::Tuple(tup) => {
                 if let [elt] = &**tup {

From 8649bf6f8f2e4b9126332578b2a2c63859b610b8 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Fri, 17 Feb 2023 22:09:18 -0500
Subject: [PATCH 418/459] Add support for match statements to parser

---
 parser/build.rs                               |    2 +
 parser/python.lalrpop                         |  562 +-
 parser/src/lib.rs                             |    1 +
 parser/src/parser.rs                          |  205 +-
 ...r__parser__tests__match_as_identifier.snap | 1674 ++++
 ..._parser__parser__tests__match_complex.snap | 8254 +++++++++++++++++
 parser/src/soft_keywords.rs                   |  117 +
 parser/src/token.rs                           |    4 +
 8 files changed, 10815 insertions(+), 4 deletions(-)
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__match_complex.snap
 create mode 100644 parser/src/soft_keywords.rs

diff --git a/parser/build.rs b/parser/build.rs
index f9a3439b..e693601d 100644
--- a/parser/build.rs
+++ b/parser/build.rs
@@ -119,6 +119,7 @@ fn gen_phf(out_dir: &Path) {
         .entry("async", "Tok::Async")
         .entry("await", "Tok::Await")
         .entry("break", "Tok::Break")
+        .entry("case", "Tok::Case")
         .entry("class", "Tok::Class")
         .entry("continue", "Tok::Continue")
         .entry("def", "Tok::Def")
@@ -135,6 +136,7 @@ fn gen_phf(out_dir: &Path) {
         .entry("in", "Tok::In")
         .entry("is", "Tok::Is")
         .entry("lambda", "Tok::Lambda")
+        .entry("match", "Tok::Match")
         .entry("nonlocal", "Tok::Nonlocal")
         .entry("not", "Tok::Not")
         .entry("or", "Tok::Or")
diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 78b71361..d9164f78 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -348,6 +348,7 @@ AssertStatement: ast::Stmt = {
 };
 
 CompoundStatement: ast::Stmt = {
+    MatchStatement,
     IfStatement,
     WhileStatement,
     ForStatement,
@@ -357,6 +358,561 @@ CompoundStatement: ast::Stmt = {
     ClassDef,
 };
 
+MatchStatement: ast::Stmt = {
+    <location:@L> "match" <subject:TestOrStarNamedExpr> ":" "\n" Indent <cases:MatchCase+> Dedent <end_location:@R> => {
+        ast::Stmt {
+            location,
+            end_location: Some(end_location),
+            custom: (),
+            node: ast::StmtKind::Match {
+                subject: Box::new(subject),
+                cases
+            }
+        }
+    },
+    <location:@L> "match" <subject:TestOrStarNamedExpr> "," ":" "\n" Indent <cases:MatchCase+> Dedent <end_location:@R> => {
+        ast::Stmt {
+            location,
+            end_location: Some(end_location),
+            custom: (),
+            node: ast::StmtKind::Match {
+                subject: Box::new(subject),
+                cases
+            }
+        }
+    },
+    <location:@L> "match" <subject:TestOrStarNamedExpr> "," <subjects:OneOrMore<TestOrStarNamedExpr>> ","? ":" "\n" Indent <cases:MatchCase+> Dedent <end_location:@R> => {
+        let mut subjects = subjects;
+        subjects.insert(0, subject);
+        ast::Stmt {
+            custom: (),
+            location,
+            end_location: Some(end_location),
+            node: ast::StmtKind::Match {
+                subject: Box::new(ast::Expr {
+                    location,
+                    end_location: Some(end_location),
+                    custom: (),
+                    node: ast::ExprKind::Tuple {
+                        elts: subjects,
+                        ctx: ast::ExprContext::Load,
+                    },
+                }),
+                cases
+            }
+        }
+    }
+}
+
+MatchCase: ast::MatchCase = {
+    "case" <pattern:Patterns> <guard:(Guard)?> ":" <body:Suite> => {
+        ast::MatchCase {
+            pattern,
+            guard: guard.map(Box::new),
+            body
+        }
+    },
+}
+
+Guard: ast::Expr = {
+    "if" <guard:NamedExpressionTest> => {
+        guard
+    }
+}
+
+Patterns: ast::Pattern = {
+    <location:@L> <pattern:Pattern> "," <end_location:@R> => ast::Pattern {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::PatternKind::MatchSequence {
+            patterns: vec![pattern]
+        },
+    },
+    <location:@L> <pattern:Pattern> "," <patterns:OneOrMore<Pattern>> ","? <end_location:@R> => {
+        let mut patterns = patterns;
+        patterns.insert(0, pattern);
+        ast::Pattern {
+            location,
+            end_location: Some(end_location),
+            custom: (),
+            node: ast::PatternKind::MatchSequence {
+                patterns
+            },
+        }
+    },
+    <pattern:Pattern> => pattern
+}
+
+Pattern: ast::Pattern = {
+    <pattern:AsPattern> => pattern,
+    <pattern:OrPattern> => pattern,
+}
+
+AsPattern: ast::Pattern = {
+    <location:@L> <pattern:OrPattern> "as" <name:Identifier> <end_location:@R> =>? {
+        if name == "_" {
+            Err(LexicalError {
+                error: LexicalErrorType::OtherError("cannot use '_' as a target".to_string()),
+                location,
+            })?
+        } else {
+            Ok(ast::Pattern {
+                location,
+                end_location: Some(end_location),
+                custom: (),
+                node: ast::PatternKind::MatchAs {
+                   pattern: Some(Box::new(pattern)),
+                   name: Some(name),
+               },
+            })
+        }
+    },
+}
+
+OrPattern: ast::Pattern = {
+    <pattern:ClosedPattern> => pattern,
+    <location:@L> <pattern:ClosedPattern> <patterns:("|" <ClosedPattern>)+> <end_location:@R> => {
+        let mut patterns = patterns;
+        patterns.insert(0, pattern);
+        ast::Pattern {
+            location,
+            end_location: Some(end_location),
+            custom: (),
+            node: ast::PatternKind::MatchOr { patterns }
+        }
+    }
+}
+
+ClosedPattern: ast::Pattern = {
+    <location:@L> <node:LiteralPattern> <end_location:@R> => ast::Pattern {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node,
+    },
+    <location:@L> <node:CapturePattern> <end_location:@R> => ast::Pattern {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node,
+    },
+    <location:@L> <node:StarPattern> <end_location:@R> => ast::Pattern {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node,
+    },
+    <location:@L> <node:ValuePattern> <end_location:@R> => ast::Pattern {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node,
+    },
+    <location:@L> <node:SequencePattern> <end_location:@R> => ast::Pattern {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node,
+    },
+    <location:@L> <node:MappingPattern> <end_location:@R> => ast::Pattern {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node,
+    },
+    <location:@L> <node:ClassPattern> <end_location:@R> => ast::Pattern {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node,
+    },
+}
+
+SequencePattern: ast::PatternKind = {
+    // A single-item tuple is a special case: it's a group pattern, _not_ a sequence pattern.
+    <location:@L> "(" <pattern:Pattern> ")" <end_location:@R> => pattern.node,
+    <location:@L> "(" ")" <end_location:@R> => ast::PatternKind::MatchSequence {
+        patterns: vec![],
+    },
+    <location:@L> "(" <pattern:Pattern> "," <patterns:Comma<Pattern>> ")" <end_location:@R> => {
+        let mut patterns = patterns;
+        patterns.insert(0, pattern);
+        ast::PatternKind::MatchSequence {
+            patterns
+        }
+    },
+    <location:@L> "[" <patterns:Comma<Pattern>> "]" <end_location:@R> => ast::PatternKind::MatchSequence {
+        patterns
+    },
+}
+
+StarPattern: ast::PatternKind = {
+    <location:@L> "*" <name:Identifier> <end_location:@R> => ast::PatternKind::MatchStar {
+        name: if name == "_" { None } else { Some(name) }
+    },
+}
+
+LiteralPattern: ast::PatternKind = {
+    "None" => ast::PatternKind::MatchSingleton {
+        value: ast::Constant::None
+    },
+    "True" => ast::PatternKind::MatchSingleton {
+        value: true.into()
+    },
+    "False" => ast::PatternKind::MatchSingleton {
+        value: false.into()
+    },
+    <location:@L> <value:Constant> <end_location:@R> => ast::PatternKind::MatchValue {
+        value: Box::new(ast::Expr {
+            location,
+            end_location: Some(end_location),
+            custom: (),
+            node: ast::ExprKind::Constant { value, kind: None }
+        })
+    },
+    <location:@L> "-" <value:Constant> <end_location:@R> => ast::PatternKind::MatchValue {
+        value: Box::new(ast::Expr {
+            location,
+            end_location: Some(end_location),
+            custom: (),
+            node: ast::ExprKind::UnaryOp {
+                op: ast::Unaryop::USub,
+                operand: Box::new(ast::Expr {
+                    location,
+                    end_location: Some(end_location),
+                    custom: (),
+                    node: ast::ExprKind::Constant { value, kind: None }
+                })
+            }
+        })
+    },
+    <location:@L> <left:Constant> <op:AddOp> <right:Constant> <end_location:@R> => ast::PatternKind::MatchValue {
+        value: Box::new(ast::Expr {
+            location,
+            end_location: Some(end_location),
+            custom: (),
+            node: ast::ExprKind::BinOp {
+                left: Box::new(ast::Expr {
+                    location,
+                    end_location: Some(end_location),
+                    custom: (),
+                    node: ast::ExprKind::Constant { value: left, kind: None }
+                }),
+                op,
+                right: Box::new(ast::Expr {
+                    location,
+                    end_location: Some(end_location),
+                    custom: (),
+                    node: ast::ExprKind::Constant { value: right, kind: None }
+                }),
+            }
+        })
+    },
+    <location:@L> "-" <left:Constant> <op:AddOp> <right:Constant> <end_location:@R> => ast::PatternKind::MatchValue {
+        value: Box::new(ast::Expr {
+            location,
+            end_location: Some(end_location),
+            custom: (),
+            node: ast::ExprKind::BinOp {
+                left: Box::new(ast::Expr {
+                    location,
+                    end_location: Some(end_location),
+                    custom: (),
+                    node: ast::ExprKind::UnaryOp {
+                        op: ast::Unaryop::USub,
+                        operand: Box::new(ast::Expr {
+                            location,
+                            end_location: Some(end_location),
+                            custom: (),
+                            node: ast::ExprKind::Constant { value: left, kind: None }
+                        })
+                    }
+                }),
+                op,
+                right: Box::new(ast::Expr {
+                    location,
+                    end_location: Some(end_location),
+                    custom: (),
+                    node: ast::ExprKind::Constant { value: right, kind: None }
+                }),
+            }
+        })
+    },
+    <location:@L> <s:(@L string @R)+> =>? Ok(ast::PatternKind::MatchValue {
+        value: Box::new(parse_strings(s)?)
+    }),
+}
+
+CapturePattern: ast::PatternKind = {
+    <location:@L> <name:Identifier> <end_location:@R> => ast::PatternKind::MatchAs {
+        pattern: None,
+        name: if name == "_" { None } else { Some(name) }
+    },
+}
+
+MatchName: ast::Expr = {
+    <location:@L> <name:Identifier> <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load },
+    },
+}
+
+MatchNameOrAttr: ast::Expr = {
+    <location:@L> <name:MatchName> "." <attr:Identifier> <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::Attribute {
+            value: Box::new(name),
+            attr,
+            ctx: ast::ExprContext::Load,
+        },
+    },
+    <location:@L> <e:MatchNameOrAttr> "." <attr:Identifier> <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::Attribute {
+            value: Box::new(e),
+            attr,
+            ctx: ast::ExprContext::Load,
+        }
+    }
+}
+
+ValuePattern: ast::PatternKind = {
+   <e:MatchNameOrAttr> => ast::PatternKind::MatchValue {
+       value: Box::new(e)
+   },
+}
+
+MappingKey: ast::Expr = {
+    <location:@L> "None" <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::Constant {
+            value: ast::Constant::None,
+            kind: None,
+        },
+    },
+    <location:@L> "True" <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::Constant {
+            value: true.into(),
+            kind: None,
+        },
+    },
+    <location:@L> "False" <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::Constant {
+            value: false.into(),
+            kind: None,
+        },
+    },
+    <location:@L> <value:Constant> <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::Constant { value, kind: None }
+    },
+    <location:@L> "-" <value:Constant> <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::UnaryOp {
+            op: ast::Unaryop::USub,
+            operand: Box::new(ast::Expr {
+                location,
+                end_location: Some(end_location),
+                custom: (),
+                node: ast::ExprKind::Constant { value, kind: None }
+            })
+        }
+    },
+    <location:@L> <left:Constant> <op:AddOp> <right:Constant> <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::BinOp {
+            left: Box::new(ast::Expr {
+                location,
+                end_location: Some(end_location),
+                custom: (),
+                node: ast::ExprKind::Constant { value: left, kind: None }
+            }),
+            op,
+            right: Box::new(ast::Expr {
+                location,
+                end_location: Some(end_location),
+                custom: (),
+                node: ast::ExprKind::Constant { value: right, kind: None }
+            }),
+        }
+    },
+    <location:@L> "-" <left:Constant> <op:AddOp> <right:Constant> <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::BinOp {
+            left: Box::new(ast::Expr {
+                location,
+                end_location: Some(end_location),
+                custom: (),
+                node: ast::ExprKind::UnaryOp {
+                    op: ast::Unaryop::USub,
+                    operand: Box::new(ast::Expr {
+                        location,
+                        end_location: Some(end_location),
+                        custom: (),
+                        node: ast::ExprKind::Constant { value: left, kind: None }
+                    })
+                }
+            }),
+            op,
+            right: Box::new(ast::Expr {
+                location,
+                end_location: Some(end_location),
+                custom: (),
+                node: ast::ExprKind::Constant { value: right, kind: None }
+            }),
+        }
+    },
+    <location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?),
+    MatchNameOrAttr,
+}
+
+MatchMappingEntry: (ast::Expr, ast::Pattern) = {
+    <k:MappingKey> ":" <v:Pattern> => (k, v),
+};
+
+MappingPattern: ast::PatternKind = {
+    <location:@L> "{" "}" <end_location:@R> => {
+        return ast::PatternKind::MatchMapping {
+            keys: vec![],
+            patterns: vec![],
+            rest: None,
+        };
+    },
+    <location:@L> "{" <e:OneOrMore<MatchMappingEntry>> "}" <end_location:@R> => {
+        let (keys, patterns) = e
+            .into_iter()
+            .unzip();
+        return ast::PatternKind::MatchMapping {
+            keys,
+            patterns,
+            rest: None,
+        };
+    },
+    <location:@L> "{" "**" <rest:Identifier> "}" <end_location:@R> => {
+        return ast::PatternKind::MatchMapping {
+            keys: vec![],
+            patterns: vec![],
+            rest: Some(rest),
+        };
+    },
+    <location:@L> "{" <e:OneOrMore<MatchMappingEntry>> ","  "**" <rest:Identifier> "}" <end_location:@R> => {
+        let (keys, patterns) = e
+            .into_iter()
+            .unzip();
+        return ast::PatternKind::MatchMapping {
+            keys,
+            patterns,
+            rest: Some(rest),
+        };
+    },
+}
+
+MatchKeywordEntry: (String, ast::Pattern) = {
+    <k:Identifier> "=" <v:Pattern> => (k, v),
+};
+
+ClassPattern: ast::PatternKind = {
+    <location:@L> <e:MatchName> "(" <patterns: OneOrMore<Pattern>> "," <kwds:OneOrMore<MatchKeywordEntry>> ","? ")" <end_location:@R> => {
+        let (kwd_attrs, kwd_patterns) = kwds
+            .into_iter()
+            .unzip();
+        ast::PatternKind::MatchClass {
+            cls: Box::new(e),
+            patterns,
+            kwd_attrs,
+            kwd_patterns,
+        }
+    },
+    <location:@L> <e:MatchName> "(" <patterns: OneOrMore<Pattern>> ","? ")" <end_location:@R> => {
+        ast::PatternKind::MatchClass {
+            cls: Box::new(e),
+            patterns,
+            kwd_attrs: vec![],
+            kwd_patterns: vec![],
+        }
+    },
+    <location:@L> <e:MatchName> "(" <kwds:OneOrMore<MatchKeywordEntry>> ","? ")" <end_location:@R> => {
+         let (kwd_attrs, kwd_patterns) = kwds
+            .into_iter()
+            .unzip();
+        ast::PatternKind::MatchClass {
+            cls: Box::new(e),
+            patterns: vec![],
+            kwd_attrs,
+            kwd_patterns,
+        }
+    },
+    <location:@L> <e:MatchName> "(" ")" <end_location:@R> => {
+        ast::PatternKind::MatchClass {
+            cls: Box::new(e),
+            patterns: vec![],
+            kwd_attrs: vec![],
+            kwd_patterns: vec![],
+        }
+    },
+    <location:@L> <e:MatchNameOrAttr> "(" <patterns: OneOrMore<Pattern>> "," <kwds:OneOrMore<MatchKeywordEntry>> ","? ")" <end_location:@R> => {
+        let (kwd_attrs, kwd_patterns) = kwds
+            .into_iter()
+            .unzip();
+        ast::PatternKind::MatchClass {
+            cls: Box::new(e),
+            patterns,
+            kwd_attrs,
+            kwd_patterns,
+        }
+    },
+    <location:@L> <e:MatchNameOrAttr> "(" <patterns: OneOrMore<Pattern>> ","? ")" <end_location:@R> => {
+        ast::PatternKind::MatchClass {
+            cls: Box::new(e),
+            patterns,
+            kwd_attrs: vec![],
+            kwd_patterns: vec![],
+        }
+    },
+    <location:@L> <e:MatchNameOrAttr> "(" <kwds:OneOrMore<MatchKeywordEntry>> ","? ")" <end_location:@R> => {
+         let (kwd_attrs, kwd_patterns) = kwds
+            .into_iter()
+            .unzip();
+        ast::PatternKind::MatchClass {
+            cls: Box::new(e),
+            patterns: vec![],
+            kwd_attrs,
+            kwd_patterns,
+        }
+    },
+    <location:@L> <e:MatchNameOrAttr> "(" ")" <end_location:@R> => {
+        ast::PatternKind::MatchClass {
+            cls: Box::new(e),
+            patterns: vec![],
+            kwd_attrs: vec![],
+            kwd_patterns: vec![],
+        }
+    },
+}
+
 IfStatement: ast::Stmt = {
     <location:@L> "if" <test:NamedExpressionTest> ":" <body:Suite> <s2:(@L "elif" NamedExpressionTest ":" Suite)*> <s3:("else" ":" Suite)?> => {
         // Determine last else:
@@ -685,7 +1241,7 @@ ParameterListStarArgs<ArgType>: (Option<Box<ast::Arg>>, Vec<ast::Arg>, Vec<ast::
         if va.is_none() && kwargs.is_empty() && kwarg.is_none() {
             Err(LexicalError {
                 error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()),
-                location: location,
+                location,
             })?
         }
 
@@ -1132,7 +1688,7 @@ Atom<Goal>: ast::Expr = {
     "(" <location:@L> "**" <e:Expression<"all">> ")" <end_location:@R> =>? {
         Err(LexicalError{
             error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()),
-            location: location,
+            location,
         }.into())
     },
     <location:@L> "{" <e:DictLiteralValues?> "}" <end_location:@R> => {
@@ -1414,6 +1970,8 @@ extern {
         "return" => lexer::Tok::Return,
         "try" => lexer::Tok::Try,
         "while" => lexer::Tok::While,
+        "match" => lexer::Tok::Match,
+        "case" => lexer::Tok::Case,
         "with" => lexer::Tok::With,
         "yield" => lexer::Tok::Yield,
         "True" => lexer::Tok::True,
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index 5c7c9640..4715dd96 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -131,4 +131,5 @@ mod string;
 #[rustfmt::skip]
 mod python;
 mod context;
+mod soft_keywords;
 pub mod token;
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 7f828021..d8f09cd4 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -18,6 +18,7 @@ use crate::{ast, error::ParseError, lexer, python};
 use ast::Location;
 use itertools::Itertools;
 use std::iter;
+use crate::soft_keywords::soft_keywords;
 
 /// Parse a full Python program usually consisting of multiple lines.
 ///  
@@ -188,9 +189,8 @@ pub fn parse_tokens(
     let tokenizer = iter::once(Ok(marker_token))
         .chain(lxr)
         .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
-
     python::TopParser::new()
-        .parse(tokenizer)
+        .parse(soft_keywords(tokenizer, mode).into_iter())
         .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
 }
 
@@ -419,4 +419,205 @@ with (0 as a, 1 as b,): pass
         assert!(parse(&source, Mode::Module, "<embedded>").is_ok());
         assert!(parse(&source, Mode::Interactive, "<embedded>").is_ok());
     }
+
+    #[test]
+    fn test_match_as_identifier() {
+        let parse_ast = parse_program(
+            r#"
+match *a + b, c   # ((match * a) + b), c
+match *(a + b), c   # (match * (a + b)), c
+match (*a + b, c)   # match ((*(a + b)), c)
+match -a * b + c   # (match - (a * b)) + c
+match -(a * b) + c   # (match - (a * b)) + c
+match (-a) * b + c   # (match (-(a * b))) + c
+match ().a   # (match()).a
+match (()).a   # (match(())).a
+match ((),).a   # (match(())).a
+match [a].b   # (match[a]).b
+match [a,].b   # (match[(a,)]).b  (not (match[a]).b)
+match [(a,)].b   # (match[(a,)]).b
+match()[a:
+    b]  # (match())[a: b]
+if match := 1: pass
+match match:
+    case 1: pass
+    case 2:
+        pass
+"#,
+            "<test>",
+        )
+        .unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_match_complex() {
+        let source = r#"# Cases sampled from Lib/test/test_patma.py
+
+# case test_patma_098
+match x:
+    case -0j:
+        y = 0
+# case test_patma_142
+match x:
+    case bytes(z):
+        y = 0
+# case test_patma_073
+match x:
+    case 0 if 0:
+        y = 0
+    case 0 if 1:
+        y = 1
+# case test_patma_006
+match 3:
+    case 0 | 1 | 2 | 3:
+        x = True
+# case test_patma_049
+match x:
+    case [0, 1] | [1, 0]:
+        y = 0
+# case black_check_sequence_then_mapping
+match x:
+    case [*_]:
+        return "seq"
+    case {}:
+        return "map"
+# case test_patma_035
+match x:
+    case {0: [1, 2, {}]}:
+        y = 0
+    case {0: [1, 2, {}] | True} | {1: [[]]} | {0: [1, 2, {}]} | [] | "X" | {}:
+        y = 1
+    case []:
+        y = 2
+# case test_patma_107
+match x:
+    case 0.25 + 1.75j:
+        y = 0
+# case test_patma_097
+match x:
+    case -0j:
+        y = 0
+# case test_patma_007
+match 4:
+    case 0 | 1 | 2 | 3:
+        x = True
+# case test_patma_154
+match x:
+    case 0 if x:
+        y = 0
+# case test_patma_134
+match x:
+    case {1: 0}:
+        y = 0
+    case {0: 0}:
+        y = 1
+    case {**z}:
+        y = 2
+# case test_patma_185
+match Seq():
+    case [*_]:
+        y = 0
+# case test_patma_063
+match x:
+    case 1:
+        y = 0
+    case 1:
+        y = 1
+# case test_patma_248
+match x:
+    case {"foo": bar}:
+        y = bar
+# case test_patma_019
+match (0, 1, 2):
+    case [0, 1, *x, 2]:
+        y = 0
+# case test_patma_052
+match x:
+    case [0]:
+        y = 0
+    case [1, 0] if (x := x[:0]):
+        y = 1
+    case [1, 0]:
+        y = 2
+# case test_patma_191
+match w:
+    case [x, y, *_]:
+        z = 0
+# case test_patma_110
+match x:
+    case -0.25 - 1.75j:
+        y = 0
+# case test_patma_151
+match (x,):
+    case [y]:
+        z = 0
+# case test_patma_114
+match x:
+    case A.B.C.D:
+        y = 0
+# case test_patma_232
+match x:
+    case None:
+        y = 0
+# case test_patma_058
+match x:
+    case 0:
+        y = 0
+# case test_patma_233
+match x:
+    case False:
+        y = 0
+# case test_patma_078
+match x:
+    case []:
+        y = 0
+    case [""]:
+        y = 1
+    case "":
+        y = 2
+# case test_patma_156
+match x:
+    case z:
+        y = 0
+# case test_patma_189
+match w:
+    case [x, y, *rest]:
+        z = 0
+# case test_patma_042
+match x:
+    case (0 as z) | (1 as z) | (2 as z) if z == x % 2:
+        y = 0
+# case test_patma_034
+match x:
+    case {0: [1, 2, {}]}:
+        y = 0
+    case {0: [1, 2, {}] | False} | {1: [[]]} | {0: [1, 2, {}]} | [] | "X" | {}:
+        y = 1
+    case []:
+        y = 2
+# case test_patma_123
+match (0, 1, 2):
+    case 0, *x:
+        y = 0
+# case test_patma_126
+match (0, 1, 2):
+    case *x, 2,:
+        y = 0
+# case test_patma_151
+match x,:
+    case y,:
+        z = 0
+# case test_patma_152
+match w, x:
+    case y, z:
+        v = 0
+# case test_patma_153
+match w := x,:
+    case y as v,:
+        z = 0
+"#;
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
 }
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap b/parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap
new file mode 100644
index 00000000..2568cc78
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap
@@ -0,0 +1,1674 @@
+---
+source: compiler/parser/src/parser.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 2,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 2,
+                column: 15,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 2,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 2,
+                        column: 15,
+                    },
+                ),
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            location: Location {
+                                row: 2,
+                                column: 0,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 12,
+                                },
+                            ),
+                            custom: (),
+                            node: BinOp {
+                                left: Located {
+                                    location: Location {
+                                        row: 2,
+                                        column: 0,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 2,
+                                            column: 8,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: BinOp {
+                                        left: Located {
+                                            location: Location {
+                                                row: 2,
+                                                column: 0,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 2,
+                                                    column: 5,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "match",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        op: Mult,
+                                        right: Located {
+                                            location: Location {
+                                                row: 2,
+                                                column: 7,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 2,
+                                                    column: 8,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "a",
+                                                ctx: Load,
+                                            },
+                                        },
+                                    },
+                                },
+                                op: Add,
+                                right: Located {
+                                    location: Location {
+                                        row: 2,
+                                        column: 11,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 2,
+                                            column: 12,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "b",
+                                        ctx: Load,
+                                    },
+                                },
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 2,
+                                column: 14,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 15,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "c",
+                                ctx: Load,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 3,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 3,
+                column: 17,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 3,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 3,
+                        column: 17,
+                    },
+                ),
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            location: Location {
+                                row: 3,
+                                column: 0,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 3,
+                                    column: 14,
+                                },
+                            ),
+                            custom: (),
+                            node: BinOp {
+                                left: Located {
+                                    location: Location {
+                                        row: 3,
+                                        column: 0,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 3,
+                                            column: 5,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "match",
+                                        ctx: Load,
+                                    },
+                                },
+                                op: Mult,
+                                right: Located {
+                                    location: Location {
+                                        row: 3,
+                                        column: 8,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 3,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: BinOp {
+                                        left: Located {
+                                            location: Location {
+                                                row: 3,
+                                                column: 8,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 3,
+                                                    column: 9,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "a",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        op: Add,
+                                        right: Located {
+                                            location: Location {
+                                                row: 3,
+                                                column: 12,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 3,
+                                                    column: 13,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "b",
+                                                ctx: Load,
+                                            },
+                                        },
+                                    },
+                                },
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 3,
+                                column: 16,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 3,
+                                    column: 17,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "c",
+                                ctx: Load,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 4,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 4,
+                column: 17,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 4,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 4,
+                        column: 17,
+                    },
+                ),
+                custom: (),
+                node: Call {
+                    func: Located {
+                        location: Location {
+                            row: 4,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 4,
+                                column: 5,
+                            },
+                        ),
+                        custom: (),
+                        node: Name {
+                            id: "match",
+                            ctx: Load,
+                        },
+                    },
+                    args: [
+                        Located {
+                            location: Location {
+                                row: 4,
+                                column: 7,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 4,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Starred {
+                                value: Located {
+                                    location: Location {
+                                        row: 4,
+                                        column: 8,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 4,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: BinOp {
+                                        left: Located {
+                                            location: Location {
+                                                row: 4,
+                                                column: 8,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 4,
+                                                    column: 9,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "a",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        op: Add,
+                                        right: Located {
+                                            location: Location {
+                                                row: 4,
+                                                column: 12,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 4,
+                                                    column: 13,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "b",
+                                                ctx: Load,
+                                            },
+                                        },
+                                    },
+                                },
+                                ctx: Load,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 4,
+                                column: 15,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 4,
+                                    column: 16,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "c",
+                                ctx: Load,
+                            },
+                        },
+                    ],
+                    keywords: [],
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 5,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 5,
+                column: 16,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 5,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 5,
+                        column: 16,
+                    },
+                ),
+                custom: (),
+                node: BinOp {
+                    left: Located {
+                        location: Location {
+                            row: 5,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 5,
+                                column: 12,
+                            },
+                        ),
+                        custom: (),
+                        node: BinOp {
+                            left: Located {
+                                location: Location {
+                                    row: 5,
+                                    column: 0,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 5,
+                                        column: 5,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "match",
+                                    ctx: Load,
+                                },
+                            },
+                            op: Sub,
+                            right: Located {
+                                location: Location {
+                                    row: 5,
+                                    column: 7,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 5,
+                                        column: 12,
+                                    },
+                                ),
+                                custom: (),
+                                node: BinOp {
+                                    left: Located {
+                                        location: Location {
+                                            row: 5,
+                                            column: 7,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 5,
+                                                column: 8,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "a",
+                                            ctx: Load,
+                                        },
+                                    },
+                                    op: Mult,
+                                    right: Located {
+                                        location: Location {
+                                            row: 5,
+                                            column: 11,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 5,
+                                                column: 12,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "b",
+                                            ctx: Load,
+                                        },
+                                    },
+                                },
+                            },
+                        },
+                    },
+                    op: Add,
+                    right: Located {
+                        location: Location {
+                            row: 5,
+                            column: 15,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 5,
+                                column: 16,
+                            },
+                        ),
+                        custom: (),
+                        node: Name {
+                            id: "c",
+                            ctx: Load,
+                        },
+                    },
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 6,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 6,
+                column: 18,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 6,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 6,
+                        column: 18,
+                    },
+                ),
+                custom: (),
+                node: BinOp {
+                    left: Located {
+                        location: Location {
+                            row: 6,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 6,
+                                column: 14,
+                            },
+                        ),
+                        custom: (),
+                        node: BinOp {
+                            left: Located {
+                                location: Location {
+                                    row: 6,
+                                    column: 0,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 6,
+                                        column: 5,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "match",
+                                    ctx: Load,
+                                },
+                            },
+                            op: Sub,
+                            right: Located {
+                                location: Location {
+                                    row: 6,
+                                    column: 8,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 6,
+                                        column: 13,
+                                    },
+                                ),
+                                custom: (),
+                                node: BinOp {
+                                    left: Located {
+                                        location: Location {
+                                            row: 6,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 6,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "a",
+                                            ctx: Load,
+                                        },
+                                    },
+                                    op: Mult,
+                                    right: Located {
+                                        location: Location {
+                                            row: 6,
+                                            column: 12,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 6,
+                                                column: 13,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "b",
+                                            ctx: Load,
+                                        },
+                                    },
+                                },
+                            },
+                        },
+                    },
+                    op: Add,
+                    right: Located {
+                        location: Location {
+                            row: 6,
+                            column: 17,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 6,
+                                column: 18,
+                            },
+                        ),
+                        custom: (),
+                        node: Name {
+                            id: "c",
+                            ctx: Load,
+                        },
+                    },
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 7,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 7,
+                column: 18,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 7,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 7,
+                        column: 18,
+                    },
+                ),
+                custom: (),
+                node: BinOp {
+                    left: Located {
+                        location: Location {
+                            row: 7,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 7,
+                                column: 14,
+                            },
+                        ),
+                        custom: (),
+                        node: BinOp {
+                            left: Located {
+                                location: Location {
+                                    row: 7,
+                                    column: 0,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 7,
+                                        column: 10,
+                                    },
+                                ),
+                                custom: (),
+                                node: Call {
+                                    func: Located {
+                                        location: Location {
+                                            row: 7,
+                                            column: 0,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 7,
+                                                column: 5,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "match",
+                                            ctx: Load,
+                                        },
+                                    },
+                                    args: [
+                                        Located {
+                                            location: Location {
+                                                row: 7,
+                                                column: 7,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 7,
+                                                    column: 9,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: UnaryOp {
+                                                op: USub,
+                                                operand: Located {
+                                                    location: Location {
+                                                        row: 7,
+                                                        column: 8,
+                                                    },
+                                                    end_location: Some(
+                                                        Location {
+                                                            row: 7,
+                                                            column: 9,
+                                                        },
+                                                    ),
+                                                    custom: (),
+                                                    node: Name {
+                                                        id: "a",
+                                                        ctx: Load,
+                                                    },
+                                                },
+                                            },
+                                        },
+                                    ],
+                                    keywords: [],
+                                },
+                            },
+                            op: Mult,
+                            right: Located {
+                                location: Location {
+                                    row: 7,
+                                    column: 13,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 7,
+                                        column: 14,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "b",
+                                    ctx: Load,
+                                },
+                            },
+                        },
+                    },
+                    op: Add,
+                    right: Located {
+                        location: Location {
+                            row: 7,
+                            column: 17,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 7,
+                                column: 18,
+                            },
+                        ),
+                        custom: (),
+                        node: Name {
+                            id: "c",
+                            ctx: Load,
+                        },
+                    },
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 8,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 8,
+                column: 10,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 8,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 8,
+                        column: 10,
+                    },
+                ),
+                custom: (),
+                node: Attribute {
+                    value: Located {
+                        location: Location {
+                            row: 8,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 8,
+                                column: 8,
+                            },
+                        ),
+                        custom: (),
+                        node: Call {
+                            func: Located {
+                                location: Location {
+                                    row: 8,
+                                    column: 0,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 8,
+                                        column: 5,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "match",
+                                    ctx: Load,
+                                },
+                            },
+                            args: [],
+                            keywords: [],
+                        },
+                    },
+                    attr: "a",
+                    ctx: Load,
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 9,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 9,
+                column: 12,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 9,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 9,
+                        column: 12,
+                    },
+                ),
+                custom: (),
+                node: Attribute {
+                    value: Located {
+                        location: Location {
+                            row: 9,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 9,
+                                column: 10,
+                            },
+                        ),
+                        custom: (),
+                        node: Call {
+                            func: Located {
+                                location: Location {
+                                    row: 9,
+                                    column: 0,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 9,
+                                        column: 5,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "match",
+                                    ctx: Load,
+                                },
+                            },
+                            args: [
+                                Located {
+                                    location: Location {
+                                        row: 9,
+                                        column: 7,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 9,
+                                            column: 9,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Tuple {
+                                        elts: [],
+                                        ctx: Load,
+                                    },
+                                },
+                            ],
+                            keywords: [],
+                        },
+                    },
+                    attr: "a",
+                    ctx: Load,
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 10,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 10,
+                column: 13,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 10,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 10,
+                        column: 13,
+                    },
+                ),
+                custom: (),
+                node: Attribute {
+                    value: Located {
+                        location: Location {
+                            row: 10,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 10,
+                                column: 11,
+                            },
+                        ),
+                        custom: (),
+                        node: Call {
+                            func: Located {
+                                location: Location {
+                                    row: 10,
+                                    column: 0,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 10,
+                                        column: 5,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "match",
+                                    ctx: Load,
+                                },
+                            },
+                            args: [
+                                Located {
+                                    location: Location {
+                                        row: 10,
+                                        column: 7,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 10,
+                                            column: 9,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Tuple {
+                                        elts: [],
+                                        ctx: Load,
+                                    },
+                                },
+                            ],
+                            keywords: [],
+                        },
+                    },
+                    attr: "a",
+                    ctx: Load,
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 11,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 11,
+                column: 11,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 11,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 11,
+                        column: 11,
+                    },
+                ),
+                custom: (),
+                node: Attribute {
+                    value: Located {
+                        location: Location {
+                            row: 11,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 11,
+                                column: 9,
+                            },
+                        ),
+                        custom: (),
+                        node: Subscript {
+                            value: Located {
+                                location: Location {
+                                    row: 11,
+                                    column: 0,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 11,
+                                        column: 5,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "match",
+                                    ctx: Load,
+                                },
+                            },
+                            slice: Located {
+                                location: Location {
+                                    row: 11,
+                                    column: 7,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 11,
+                                        column: 8,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "a",
+                                    ctx: Load,
+                                },
+                            },
+                            ctx: Load,
+                        },
+                    },
+                    attr: "b",
+                    ctx: Load,
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 12,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 12,
+                column: 12,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 12,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 12,
+                        column: 12,
+                    },
+                ),
+                custom: (),
+                node: Attribute {
+                    value: Located {
+                        location: Location {
+                            row: 12,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 12,
+                                column: 10,
+                            },
+                        ),
+                        custom: (),
+                        node: Subscript {
+                            value: Located {
+                                location: Location {
+                                    row: 12,
+                                    column: 0,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 12,
+                                        column: 5,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "match",
+                                    ctx: Load,
+                                },
+                            },
+                            slice: Located {
+                                location: Location {
+                                    row: 12,
+                                    column: 7,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 12,
+                                        column: 9,
+                                    },
+                                ),
+                                custom: (),
+                                node: Tuple {
+                                    elts: [
+                                        Located {
+                                            location: Location {
+                                                row: 12,
+                                                column: 7,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 12,
+                                                    column: 8,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "a",
+                                                ctx: Load,
+                                            },
+                                        },
+                                    ],
+                                    ctx: Load,
+                                },
+                            },
+                            ctx: Load,
+                        },
+                    },
+                    attr: "b",
+                    ctx: Load,
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 13,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 13,
+                column: 14,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 13,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 13,
+                        column: 14,
+                    },
+                ),
+                custom: (),
+                node: Attribute {
+                    value: Located {
+                        location: Location {
+                            row: 13,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 13,
+                                column: 12,
+                            },
+                        ),
+                        custom: (),
+                        node: Subscript {
+                            value: Located {
+                                location: Location {
+                                    row: 13,
+                                    column: 0,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 13,
+                                        column: 5,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "match",
+                                    ctx: Load,
+                                },
+                            },
+                            slice: Located {
+                                location: Location {
+                                    row: 13,
+                                    column: 7,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 13,
+                                        column: 11,
+                                    },
+                                ),
+                                custom: (),
+                                node: Tuple {
+                                    elts: [
+                                        Located {
+                                            location: Location {
+                                                row: 13,
+                                                column: 8,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 13,
+                                                    column: 9,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "a",
+                                                ctx: Load,
+                                            },
+                                        },
+                                    ],
+                                    ctx: Load,
+                                },
+                            },
+                            ctx: Load,
+                        },
+                    },
+                    attr: "b",
+                    ctx: Load,
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 14,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 15,
+                column: 6,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 14,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 15,
+                        column: 6,
+                    },
+                ),
+                custom: (),
+                node: Subscript {
+                    value: Located {
+                        location: Location {
+                            row: 14,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 14,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Call {
+                            func: Located {
+                                location: Location {
+                                    row: 14,
+                                    column: 0,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 14,
+                                        column: 5,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "match",
+                                    ctx: Load,
+                                },
+                            },
+                            args: [],
+                            keywords: [],
+                        },
+                    },
+                    slice: Located {
+                        location: Location {
+                            row: 14,
+                            column: 8,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 15,
+                                column: 5,
+                            },
+                        ),
+                        custom: (),
+                        node: Slice {
+                            lower: Some(
+                                Located {
+                                    location: Location {
+                                        row: 14,
+                                        column: 8,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 14,
+                                            column: 9,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "a",
+                                        ctx: Load,
+                                    },
+                                },
+                            ),
+                            upper: Some(
+                                Located {
+                                    location: Location {
+                                        row: 15,
+                                        column: 4,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 15,
+                                            column: 5,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "b",
+                                        ctx: Load,
+                                    },
+                                },
+                            ),
+                            step: None,
+                        },
+                    },
+                    ctx: Load,
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 16,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 16,
+                column: 19,
+            },
+        ),
+        custom: (),
+        node: If {
+            test: Located {
+                location: Location {
+                    row: 16,
+                    column: 3,
+                },
+                end_location: Some(
+                    Location {
+                        row: 16,
+                        column: 13,
+                    },
+                ),
+                custom: (),
+                node: NamedExpr {
+                    target: Located {
+                        location: Location {
+                            row: 16,
+                            column: 3,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 16,
+                                column: 8,
+                            },
+                        ),
+                        custom: (),
+                        node: Name {
+                            id: "match",
+                            ctx: Store,
+                        },
+                    },
+                    value: Located {
+                        location: Location {
+                            row: 16,
+                            column: 12,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 16,
+                                column: 13,
+                            },
+                        ),
+                        custom: (),
+                        node: Constant {
+                            value: Int(
+                                1,
+                            ),
+                            kind: None,
+                        },
+                    },
+                },
+            },
+            body: [
+                Located {
+                    location: Location {
+                        row: 16,
+                        column: 15,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 16,
+                            column: 19,
+                        },
+                    ),
+                    custom: (),
+                    node: Pass,
+                },
+            ],
+            orelse: [],
+        },
+    },
+    Located {
+        location: Location {
+            row: 17,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 21,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 17,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 17,
+                        column: 11,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "match",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 18,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 18,
+                                column: 10,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 18,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 18,
+                                        column: 10,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        1,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 18,
+                                column: 12,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 18,
+                                    column: 16,
+                                },
+                            ),
+                            custom: (),
+                            node: Pass,
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 19,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 19,
+                                column: 10,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 19,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 19,
+                                        column: 10,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        2,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 20,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 20,
+                                    column: 12,
+                                },
+                            ),
+                            custom: (),
+                            node: Pass,
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__match_complex.snap b/parser/src/snapshots/rustpython_parser__parser__tests__match_complex.snap
new file mode 100644
index 00000000..83171725
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__match_complex.snap
@@ -0,0 +1,8254 @@
+---
+source: compiler/parser/src/parser.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 4,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 8,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 4,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 4,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 5,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 5,
+                                column: 12,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 5,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 5,
+                                        column: 12,
+                                    },
+                                ),
+                                custom: (),
+                                node: UnaryOp {
+                                    op: USub,
+                                    operand: Located {
+                                        location: Location {
+                                            row: 5,
+                                            column: 9,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 5,
+                                                column: 12,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Constant {
+                                            value: Complex {
+                                                real: 0.0,
+                                                imag: 0.0,
+                                            },
+                                            kind: None,
+                                        },
+                                    },
+                                },
+                            },
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 6,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 6,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 6,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 6,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 6,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 6,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 8,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 12,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 8,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 8,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 9,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 9,
+                                column: 17,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchClass {
+                            cls: Located {
+                                location: Location {
+                                    row: 9,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 9,
+                                        column: 14,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "bytes",
+                                    ctx: Load,
+                                },
+                            },
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 9,
+                                        column: 15,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 9,
+                                            column: 16,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: None,
+                                        name: Some(
+                                            "z",
+                                        ),
+                                    },
+                                },
+                            ],
+                            kwd_attrs: [],
+                            kwd_patterns: [],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 10,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 10,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 10,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 10,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 10,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 10,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 12,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 18,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 12,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 12,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 13,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 13,
+                                column: 10,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 13,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 13,
+                                        column: 10,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        0,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    guard: Some(
+                        Located {
+                            location: Location {
+                                row: 13,
+                                column: 14,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 13,
+                                    column: 15,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    0,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ),
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 14,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 14,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 14,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 14,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 14,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 14,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 15,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 15,
+                                column: 10,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 15,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 15,
+                                        column: 10,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        0,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    guard: Some(
+                        Located {
+                            location: Location {
+                                row: 15,
+                                column: 14,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 15,
+                                    column: 15,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ),
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 16,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 16,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 16,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 16,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 16,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 16,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            1,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 18,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 22,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 18,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 18,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Int(
+                        3,
+                    ),
+                    kind: None,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 19,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 19,
+                                column: 22,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchOr {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 19,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 19,
+                                            column: 10,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 19,
+                                                column: 9,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 19,
+                                                    column: 10,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 19,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 19,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 19,
+                                                column: 13,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 19,
+                                                    column: 14,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    1,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 19,
+                                        column: 17,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 19,
+                                            column: 18,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 19,
+                                                column: 17,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 19,
+                                                    column: 18,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    2,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 19,
+                                        column: 21,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 19,
+                                            column: 22,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 19,
+                                                column: 21,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 19,
+                                                    column: 22,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    3,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 20,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 20,
+                                    column: 16,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 20,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 20,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "x",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 20,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 20,
+                                            column: 16,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Bool(
+                                            true,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 22,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 26,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 22,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 22,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 23,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 23,
+                                column: 24,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchOr {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 23,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 23,
+                                            column: 15,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchSequence {
+                                        patterns: [
+                                            Located {
+                                                location: Location {
+                                                    row: 23,
+                                                    column: 10,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 23,
+                                                        column: 11,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchValue {
+                                                    value: Located {
+                                                        location: Location {
+                                                            row: 23,
+                                                            column: 10,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 23,
+                                                                column: 11,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Constant {
+                                                            value: Int(
+                                                                0,
+                                                            ),
+                                                            kind: None,
+                                                        },
+                                                    },
+                                                },
+                                            },
+                                            Located {
+                                                location: Location {
+                                                    row: 23,
+                                                    column: 13,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 23,
+                                                        column: 14,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchValue {
+                                                    value: Located {
+                                                        location: Location {
+                                                            row: 23,
+                                                            column: 13,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 23,
+                                                                column: 14,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Constant {
+                                                            value: Int(
+                                                                1,
+                                                            ),
+                                                            kind: None,
+                                                        },
+                                                    },
+                                                },
+                                            },
+                                        ],
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 23,
+                                        column: 18,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 23,
+                                            column: 24,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchSequence {
+                                        patterns: [
+                                            Located {
+                                                location: Location {
+                                                    row: 23,
+                                                    column: 19,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 23,
+                                                        column: 20,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchValue {
+                                                    value: Located {
+                                                        location: Location {
+                                                            row: 23,
+                                                            column: 19,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 23,
+                                                                column: 20,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Constant {
+                                                            value: Int(
+                                                                1,
+                                                            ),
+                                                            kind: None,
+                                                        },
+                                                    },
+                                                },
+                                            },
+                                            Located {
+                                                location: Location {
+                                                    row: 23,
+                                                    column: 22,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 23,
+                                                        column: 23,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchValue {
+                                                    value: Located {
+                                                        location: Location {
+                                                            row: 23,
+                                                            column: 22,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 23,
+                                                                column: 23,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Constant {
+                                                            value: Int(
+                                                                0,
+                                                            ),
+                                                            kind: None,
+                                                        },
+                                                    },
+                                                },
+                                            },
+                                        ],
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 24,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 24,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 24,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 24,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 24,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 24,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 26,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 32,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 26,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 26,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 27,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 27,
+                                column: 13,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 27,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 27,
+                                            column: 12,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchStar {
+                                        name: None,
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 28,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 28,
+                                    column: 20,
+                                },
+                            ),
+                            custom: (),
+                            node: Return {
+                                value: Some(
+                                    Located {
+                                        location: Location {
+                                            row: 28,
+                                            column: 15,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 28,
+                                                column: 20,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Constant {
+                                            value: Str(
+                                                "seq",
+                                            ),
+                                            kind: None,
+                                        },
+                                    },
+                                ),
+                            },
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 29,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 29,
+                                column: 11,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchMapping {
+                            keys: [],
+                            patterns: [],
+                            rest: None,
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 30,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 30,
+                                    column: 20,
+                                },
+                            ),
+                            custom: (),
+                            node: Return {
+                                value: Some(
+                                    Located {
+                                        location: Location {
+                                            row: 30,
+                                            column: 15,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 30,
+                                                column: 20,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Constant {
+                                            value: Str(
+                                                "map",
+                                            ),
+                                            kind: None,
+                                        },
+                                    },
+                                ),
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 32,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 40,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 32,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 32,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 33,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 33,
+                                column: 24,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchMapping {
+                            keys: [
+                                Located {
+                                    location: Location {
+                                        row: 33,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 33,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 33,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 33,
+                                            column: 23,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchSequence {
+                                        patterns: [
+                                            Located {
+                                                location: Location {
+                                                    row: 33,
+                                                    column: 14,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 33,
+                                                        column: 15,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchValue {
+                                                    value: Located {
+                                                        location: Location {
+                                                            row: 33,
+                                                            column: 14,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 33,
+                                                                column: 15,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Constant {
+                                                            value: Int(
+                                                                1,
+                                                            ),
+                                                            kind: None,
+                                                        },
+                                                    },
+                                                },
+                                            },
+                                            Located {
+                                                location: Location {
+                                                    row: 33,
+                                                    column: 17,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 33,
+                                                        column: 18,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchValue {
+                                                    value: Located {
+                                                        location: Location {
+                                                            row: 33,
+                                                            column: 17,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 33,
+                                                                column: 18,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Constant {
+                                                            value: Int(
+                                                                2,
+                                                            ),
+                                                            kind: None,
+                                                        },
+                                                    },
+                                                },
+                                            },
+                                            Located {
+                                                location: Location {
+                                                    row: 33,
+                                                    column: 20,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 33,
+                                                        column: 22,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchMapping {
+                                                    keys: [],
+                                                    patterns: [],
+                                                    rest: None,
+                                                },
+                                            },
+                                        ],
+                                    },
+                                },
+                            ],
+                            rest: None,
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 34,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 34,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 34,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 34,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 34,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 34,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 35,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 35,
+                                column: 77,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchOr {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 35,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 35,
+                                            column: 31,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchMapping {
+                                        keys: [
+                                            Located {
+                                                location: Location {
+                                                    row: 35,
+                                                    column: 10,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 35,
+                                                        column: 11,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Constant {
+                                                    value: Int(
+                                                        0,
+                                                    ),
+                                                    kind: None,
+                                                },
+                                            },
+                                        ],
+                                        patterns: [
+                                            Located {
+                                                location: Location {
+                                                    row: 35,
+                                                    column: 13,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 35,
+                                                        column: 30,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchOr {
+                                                    patterns: [
+                                                        Located {
+                                                            location: Location {
+                                                                row: 35,
+                                                                column: 13,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 35,
+                                                                    column: 23,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchSequence {
+                                                                patterns: [
+                                                                    Located {
+                                                                        location: Location {
+                                                                            row: 35,
+                                                                            column: 14,
+                                                                        },
+                                                                        end_location: Some(
+                                                                            Location {
+                                                                                row: 35,
+                                                                                column: 15,
+                                                                            },
+                                                                        ),
+                                                                        custom: (),
+                                                                        node: MatchValue {
+                                                                            value: Located {
+                                                                                location: Location {
+                                                                                    row: 35,
+                                                                                    column: 14,
+                                                                                },
+                                                                                end_location: Some(
+                                                                                    Location {
+                                                                                        row: 35,
+                                                                                        column: 15,
+                                                                                    },
+                                                                                ),
+                                                                                custom: (),
+                                                                                node: Constant {
+                                                                                    value: Int(
+                                                                                        1,
+                                                                                    ),
+                                                                                    kind: None,
+                                                                                },
+                                                                            },
+                                                                        },
+                                                                    },
+                                                                    Located {
+                                                                        location: Location {
+                                                                            row: 35,
+                                                                            column: 17,
+                                                                        },
+                                                                        end_location: Some(
+                                                                            Location {
+                                                                                row: 35,
+                                                                                column: 18,
+                                                                            },
+                                                                        ),
+                                                                        custom: (),
+                                                                        node: MatchValue {
+                                                                            value: Located {
+                                                                                location: Location {
+                                                                                    row: 35,
+                                                                                    column: 17,
+                                                                                },
+                                                                                end_location: Some(
+                                                                                    Location {
+                                                                                        row: 35,
+                                                                                        column: 18,
+                                                                                    },
+                                                                                ),
+                                                                                custom: (),
+                                                                                node: Constant {
+                                                                                    value: Int(
+                                                                                        2,
+                                                                                    ),
+                                                                                    kind: None,
+                                                                                },
+                                                                            },
+                                                                        },
+                                                                    },
+                                                                    Located {
+                                                                        location: Location {
+                                                                            row: 35,
+                                                                            column: 20,
+                                                                        },
+                                                                        end_location: Some(
+                                                                            Location {
+                                                                                row: 35,
+                                                                                column: 22,
+                                                                            },
+                                                                        ),
+                                                                        custom: (),
+                                                                        node: MatchMapping {
+                                                                            keys: [],
+                                                                            patterns: [],
+                                                                            rest: None,
+                                                                        },
+                                                                    },
+                                                                ],
+                                                            },
+                                                        },
+                                                        Located {
+                                                            location: Location {
+                                                                row: 35,
+                                                                column: 26,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 35,
+                                                                    column: 30,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchSingleton {
+                                                                value: Bool(
+                                                                    true,
+                                                                ),
+                                                            },
+                                                        },
+                                                    ],
+                                                },
+                                            },
+                                        ],
+                                        rest: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 35,
+                                        column: 34,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 35,
+                                            column: 43,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchMapping {
+                                        keys: [
+                                            Located {
+                                                location: Location {
+                                                    row: 35,
+                                                    column: 35,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 35,
+                                                        column: 36,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Constant {
+                                                    value: Int(
+                                                        1,
+                                                    ),
+                                                    kind: None,
+                                                },
+                                            },
+                                        ],
+                                        patterns: [
+                                            Located {
+                                                location: Location {
+                                                    row: 35,
+                                                    column: 38,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 35,
+                                                        column: 42,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchSequence {
+                                                    patterns: [
+                                                        Located {
+                                                            location: Location {
+                                                                row: 35,
+                                                                column: 39,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 35,
+                                                                    column: 41,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchSequence {
+                                                                patterns: [],
+                                                            },
+                                                        },
+                                                    ],
+                                                },
+                                            },
+                                        ],
+                                        rest: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 35,
+                                        column: 46,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 35,
+                                            column: 61,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchMapping {
+                                        keys: [
+                                            Located {
+                                                location: Location {
+                                                    row: 35,
+                                                    column: 47,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 35,
+                                                        column: 48,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Constant {
+                                                    value: Int(
+                                                        0,
+                                                    ),
+                                                    kind: None,
+                                                },
+                                            },
+                                        ],
+                                        patterns: [
+                                            Located {
+                                                location: Location {
+                                                    row: 35,
+                                                    column: 50,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 35,
+                                                        column: 60,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchSequence {
+                                                    patterns: [
+                                                        Located {
+                                                            location: Location {
+                                                                row: 35,
+                                                                column: 51,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 35,
+                                                                    column: 52,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchValue {
+                                                                value: Located {
+                                                                    location: Location {
+                                                                        row: 35,
+                                                                        column: 51,
+                                                                    },
+                                                                    end_location: Some(
+                                                                        Location {
+                                                                            row: 35,
+                                                                            column: 52,
+                                                                        },
+                                                                    ),
+                                                                    custom: (),
+                                                                    node: Constant {
+                                                                        value: Int(
+                                                                            1,
+                                                                        ),
+                                                                        kind: None,
+                                                                    },
+                                                                },
+                                                            },
+                                                        },
+                                                        Located {
+                                                            location: Location {
+                                                                row: 35,
+                                                                column: 54,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 35,
+                                                                    column: 55,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchValue {
+                                                                value: Located {
+                                                                    location: Location {
+                                                                        row: 35,
+                                                                        column: 54,
+                                                                    },
+                                                                    end_location: Some(
+                                                                        Location {
+                                                                            row: 35,
+                                                                            column: 55,
+                                                                        },
+                                                                    ),
+                                                                    custom: (),
+                                                                    node: Constant {
+                                                                        value: Int(
+                                                                            2,
+                                                                        ),
+                                                                        kind: None,
+                                                                    },
+                                                                },
+                                                            },
+                                                        },
+                                                        Located {
+                                                            location: Location {
+                                                                row: 35,
+                                                                column: 57,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 35,
+                                                                    column: 59,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchMapping {
+                                                                keys: [],
+                                                                patterns: [],
+                                                                rest: None,
+                                                            },
+                                                        },
+                                                    ],
+                                                },
+                                            },
+                                        ],
+                                        rest: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 35,
+                                        column: 64,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 35,
+                                            column: 66,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchSequence {
+                                        patterns: [],
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 35,
+                                        column: 69,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 35,
+                                            column: 72,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 35,
+                                                column: 69,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 35,
+                                                    column: 72,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Str(
+                                                    "X",
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 35,
+                                        column: 75,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 35,
+                                            column: 77,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchMapping {
+                                        keys: [],
+                                        patterns: [],
+                                        rest: None,
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 36,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 36,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 36,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 36,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 36,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 36,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            1,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 37,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 37,
+                                column: 11,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 38,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 38,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 38,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 38,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 38,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 38,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            2,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 40,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 44,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 40,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 40,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 41,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 41,
+                                column: 21,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 41,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 41,
+                                        column: 21,
+                                    },
+                                ),
+                                custom: (),
+                                node: BinOp {
+                                    left: Located {
+                                        location: Location {
+                                            row: 41,
+                                            column: 9,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 41,
+                                                column: 21,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Constant {
+                                            value: Float(
+                                                0.25,
+                                            ),
+                                            kind: None,
+                                        },
+                                    },
+                                    op: Add,
+                                    right: Located {
+                                        location: Location {
+                                            row: 41,
+                                            column: 9,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 41,
+                                                column: 21,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Constant {
+                                            value: Complex {
+                                                real: 0.0,
+                                                imag: 1.75,
+                                            },
+                                            kind: None,
+                                        },
+                                    },
+                                },
+                            },
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 42,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 42,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 42,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 42,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 42,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 42,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 44,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 48,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 44,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 44,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 45,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 45,
+                                column: 12,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 45,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 45,
+                                        column: 12,
+                                    },
+                                ),
+                                custom: (),
+                                node: UnaryOp {
+                                    op: USub,
+                                    operand: Located {
+                                        location: Location {
+                                            row: 45,
+                                            column: 9,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 45,
+                                                column: 12,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Constant {
+                                            value: Complex {
+                                                real: 0.0,
+                                                imag: 0.0,
+                                            },
+                                            kind: None,
+                                        },
+                                    },
+                                },
+                            },
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 46,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 46,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 46,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 46,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 46,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 46,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 48,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 52,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 48,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 48,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Int(
+                        4,
+                    ),
+                    kind: None,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 49,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 49,
+                                column: 22,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchOr {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 49,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 49,
+                                            column: 10,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 49,
+                                                column: 9,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 49,
+                                                    column: 10,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 49,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 49,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 49,
+                                                column: 13,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 49,
+                                                    column: 14,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    1,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 49,
+                                        column: 17,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 49,
+                                            column: 18,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 49,
+                                                column: 17,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 49,
+                                                    column: 18,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    2,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 49,
+                                        column: 21,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 49,
+                                            column: 22,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 49,
+                                                column: 21,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 49,
+                                                    column: 22,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    3,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 50,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 50,
+                                    column: 16,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 50,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 50,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "x",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 50,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 50,
+                                            column: 16,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Bool(
+                                            true,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 52,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 56,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 52,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 52,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 53,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 53,
+                                column: 10,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 53,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 53,
+                                        column: 10,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        0,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    guard: Some(
+                        Located {
+                            location: Location {
+                                row: 53,
+                                column: 14,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 53,
+                                    column: 15,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Load,
+                            },
+                        },
+                    ),
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 54,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 54,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 54,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 54,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 54,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 54,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 56,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 64,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 56,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 56,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 57,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 57,
+                                column: 15,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchMapping {
+                            keys: [
+                                Located {
+                                    location: Location {
+                                        row: 57,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 57,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            1,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 57,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 57,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 57,
+                                                column: 13,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 57,
+                                                    column: 14,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                            rest: None,
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 58,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 58,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 58,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 58,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 58,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 58,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 59,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 59,
+                                column: 15,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchMapping {
+                            keys: [
+                                Located {
+                                    location: Location {
+                                        row: 59,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 59,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 59,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 59,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 59,
+                                                column: 13,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 59,
+                                                    column: 14,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                            rest: None,
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 60,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 60,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 60,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 60,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 60,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 60,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            1,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 61,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 61,
+                                column: 14,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchMapping {
+                            keys: [],
+                            patterns: [],
+                            rest: Some(
+                                "z",
+                            ),
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 62,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 62,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 62,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 62,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 62,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 62,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            2,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 64,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 68,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 64,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 64,
+                        column: 11,
+                    },
+                ),
+                custom: (),
+                node: Call {
+                    func: Located {
+                        location: Location {
+                            row: 64,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 64,
+                                column: 9,
+                            },
+                        ),
+                        custom: (),
+                        node: Name {
+                            id: "Seq",
+                            ctx: Load,
+                        },
+                    },
+                    args: [],
+                    keywords: [],
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 65,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 65,
+                                column: 13,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 65,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 65,
+                                            column: 12,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchStar {
+                                        name: None,
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 66,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 66,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 66,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 66,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 66,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 66,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 68,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 74,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 68,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 68,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 69,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 69,
+                                column: 10,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 69,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 69,
+                                        column: 10,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        1,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 70,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 70,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 70,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 70,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 70,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 70,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 71,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 71,
+                                column: 10,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 71,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 71,
+                                        column: 10,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        1,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 72,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 72,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 72,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 72,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 72,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 72,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            1,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 74,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 78,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 74,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 74,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 75,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 75,
+                                column: 21,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchMapping {
+                            keys: [
+                                Located {
+                                    location: Location {
+                                        row: 75,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 75,
+                                            column: 15,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Str(
+                                            "foo",
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 75,
+                                        column: 17,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 75,
+                                            column: 20,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: None,
+                                        name: Some(
+                                            "bar",
+                                        ),
+                                    },
+                                },
+                            ],
+                            rest: None,
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 76,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 76,
+                                    column: 15,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 76,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 76,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 76,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 76,
+                                            column: 15,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "bar",
+                                        ctx: Load,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 78,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 82,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 78,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 78,
+                        column: 15,
+                    },
+                ),
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            location: Location {
+                                row: 78,
+                                column: 7,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 78,
+                                    column: 8,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    0,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 78,
+                                column: 10,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 78,
+                                    column: 11,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 78,
+                                column: 13,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 78,
+                                    column: 14,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 79,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 79,
+                                column: 22,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 79,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 79,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 79,
+                                                column: 10,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 79,
+                                                    column: 11,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 79,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 79,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 79,
+                                                column: 13,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 79,
+                                                    column: 14,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    1,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 79,
+                                        column: 16,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 79,
+                                            column: 18,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchStar {
+                                        name: Some(
+                                            "x",
+                                        ),
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 79,
+                                        column: 20,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 79,
+                                            column: 21,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 79,
+                                                column: 20,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 79,
+                                                    column: 21,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    2,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 80,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 80,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 80,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 80,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 80,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 80,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 82,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 90,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 82,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 82,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 83,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 83,
+                                column: 12,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 83,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 83,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 83,
+                                                column: 10,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 83,
+                                                    column: 11,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 84,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 84,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 84,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 84,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 84,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 84,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 85,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 85,
+                                column: 15,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 85,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 85,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 85,
+                                                column: 10,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 85,
+                                                    column: 11,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    1,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 85,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 85,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 85,
+                                                column: 13,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 85,
+                                                    column: 14,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: Some(
+                        Located {
+                            location: Location {
+                                row: 85,
+                                column: 20,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 85,
+                                    column: 30,
+                                },
+                            ),
+                            custom: (),
+                            node: NamedExpr {
+                                target: Located {
+                                    location: Location {
+                                        row: 85,
+                                        column: 20,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 85,
+                                            column: 21,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "x",
+                                        ctx: Store,
+                                    },
+                                },
+                                value: Located {
+                                    location: Location {
+                                        row: 85,
+                                        column: 25,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 85,
+                                            column: 30,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Subscript {
+                                        value: Located {
+                                            location: Location {
+                                                row: 85,
+                                                column: 25,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 85,
+                                                    column: 26,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "x",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        slice: Located {
+                                            location: Location {
+                                                row: 85,
+                                                column: 27,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 85,
+                                                    column: 29,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Slice {
+                                                lower: None,
+                                                upper: Some(
+                                                    Located {
+                                                        location: Location {
+                                                            row: 85,
+                                                            column: 28,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 85,
+                                                                column: 29,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Constant {
+                                                            value: Int(
+                                                                0,
+                                                            ),
+                                                            kind: None,
+                                                        },
+                                                    },
+                                                ),
+                                                step: None,
+                                            },
+                                        },
+                                        ctx: Load,
+                                    },
+                                },
+                            },
+                        },
+                    ),
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 86,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 86,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 86,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 86,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 86,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 86,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            1,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 87,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 87,
+                                column: 15,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 87,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 87,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 87,
+                                                column: 10,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 87,
+                                                    column: 11,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    1,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 87,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 87,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 87,
+                                                column: 13,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 87,
+                                                    column: 14,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 88,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 88,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 88,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 88,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 88,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 88,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            2,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 90,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 94,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 90,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 90,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "w",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 91,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 91,
+                                column: 19,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 91,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 91,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: None,
+                                        name: Some(
+                                            "x",
+                                        ),
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 91,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 91,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: None,
+                                        name: Some(
+                                            "y",
+                                        ),
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 91,
+                                        column: 16,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 91,
+                                            column: 18,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchStar {
+                                        name: None,
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 92,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 92,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 92,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 92,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "z",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 92,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 92,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 94,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 98,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 94,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 94,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 95,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 95,
+                                column: 22,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 95,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 95,
+                                        column: 22,
+                                    },
+                                ),
+                                custom: (),
+                                node: BinOp {
+                                    left: Located {
+                                        location: Location {
+                                            row: 95,
+                                            column: 9,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 95,
+                                                column: 22,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: UnaryOp {
+                                            op: USub,
+                                            operand: Located {
+                                                location: Location {
+                                                    row: 95,
+                                                    column: 9,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 95,
+                                                        column: 22,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Constant {
+                                                    value: Float(
+                                                        0.25,
+                                                    ),
+                                                    kind: None,
+                                                },
+                                            },
+                                        },
+                                    },
+                                    op: Sub,
+                                    right: Located {
+                                        location: Location {
+                                            row: 95,
+                                            column: 9,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 95,
+                                                column: 22,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Constant {
+                                            value: Complex {
+                                                real: 0.0,
+                                                imag: 1.75,
+                                            },
+                                            kind: None,
+                                        },
+                                    },
+                                },
+                            },
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 96,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 96,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 96,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 96,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 96,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 96,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 98,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 102,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 98,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 98,
+                        column: 10,
+                    },
+                ),
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            location: Location {
+                                row: 98,
+                                column: 7,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 98,
+                                    column: 8,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Load,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 99,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 99,
+                                column: 12,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 99,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 99,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: None,
+                                        name: Some(
+                                            "y",
+                                        ),
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 100,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 100,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 100,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 100,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "z",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 100,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 100,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 102,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 106,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 102,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 102,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 103,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 103,
+                                column: 16,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 103,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 103,
+                                        column: 16,
+                                    },
+                                ),
+                                custom: (),
+                                node: Attribute {
+                                    value: Located {
+                                        location: Location {
+                                            row: 103,
+                                            column: 9,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 103,
+                                                column: 14,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Attribute {
+                                            value: Located {
+                                                location: Location {
+                                                    row: 103,
+                                                    column: 9,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 103,
+                                                        column: 12,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Attribute {
+                                                    value: Located {
+                                                        location: Location {
+                                                            row: 103,
+                                                            column: 9,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 103,
+                                                                column: 10,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Name {
+                                                            id: "A",
+                                                            ctx: Load,
+                                                        },
+                                                    },
+                                                    attr: "B",
+                                                    ctx: Load,
+                                                },
+                                            },
+                                            attr: "C",
+                                            ctx: Load,
+                                        },
+                                    },
+                                    attr: "D",
+                                    ctx: Load,
+                                },
+                            },
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 104,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 104,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 104,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 104,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 104,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 104,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 106,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 110,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 106,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 106,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 107,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 107,
+                                column: 13,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSingleton {
+                            value: None,
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 108,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 108,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 108,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 108,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 108,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 108,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 110,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 114,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 110,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 110,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 111,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 111,
+                                column: 10,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 111,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 111,
+                                        column: 10,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Int(
+                                        0,
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 112,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 112,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 112,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 112,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 112,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 112,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 114,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 118,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 114,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 114,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 115,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 115,
+                                column: 14,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSingleton {
+                            value: Bool(
+                                false,
+                            ),
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 116,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 116,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 116,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 116,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 116,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 116,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 118,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 126,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 118,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 118,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 119,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 119,
+                                column: 11,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 120,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 120,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 120,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 120,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 120,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 120,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 121,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 121,
+                                column: 13,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 121,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 121,
+                                            column: 12,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 121,
+                                                column: 10,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 121,
+                                                    column: 12,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Str(
+                                                    "",
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 122,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 122,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 122,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 122,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 122,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 122,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            1,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 123,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 123,
+                                column: 11,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchValue {
+                            value: Located {
+                                location: Location {
+                                    row: 123,
+                                    column: 9,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 123,
+                                        column: 11,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Str(
+                                        "",
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 124,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 124,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 124,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 124,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 124,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 124,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            2,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 126,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 130,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 126,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 126,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 127,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 127,
+                                column: 10,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchAs {
+                            pattern: None,
+                            name: Some(
+                                "z",
+                            ),
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 128,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 128,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 128,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 128,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 128,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 128,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 130,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 134,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 130,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 130,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "w",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 131,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 131,
+                                column: 22,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 131,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 131,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: None,
+                                        name: Some(
+                                            "x",
+                                        ),
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 131,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 131,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: None,
+                                        name: Some(
+                                            "y",
+                                        ),
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 131,
+                                        column: 16,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 131,
+                                            column: 21,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchStar {
+                                        name: Some(
+                                            "rest",
+                                        ),
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 132,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 132,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 132,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 132,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "z",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 132,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 132,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 134,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 138,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 134,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 134,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 135,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 135,
+                                column: 39,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchOr {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 135,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 135,
+                                            column: 17,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: Some(
+                                            Located {
+                                                location: Location {
+                                                    row: 135,
+                                                    column: 10,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 135,
+                                                        column: 11,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchValue {
+                                                    value: Located {
+                                                        location: Location {
+                                                            row: 135,
+                                                            column: 10,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 135,
+                                                                column: 11,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Constant {
+                                                            value: Int(
+                                                                0,
+                                                            ),
+                                                            kind: None,
+                                                        },
+                                                    },
+                                                },
+                                            },
+                                        ),
+                                        name: Some(
+                                            "z",
+                                        ),
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 135,
+                                        column: 20,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 135,
+                                            column: 28,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: Some(
+                                            Located {
+                                                location: Location {
+                                                    row: 135,
+                                                    column: 21,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 135,
+                                                        column: 22,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchValue {
+                                                    value: Located {
+                                                        location: Location {
+                                                            row: 135,
+                                                            column: 21,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 135,
+                                                                column: 22,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Constant {
+                                                            value: Int(
+                                                                1,
+                                                            ),
+                                                            kind: None,
+                                                        },
+                                                    },
+                                                },
+                                            },
+                                        ),
+                                        name: Some(
+                                            "z",
+                                        ),
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 135,
+                                        column: 31,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 135,
+                                            column: 39,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: Some(
+                                            Located {
+                                                location: Location {
+                                                    row: 135,
+                                                    column: 32,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 135,
+                                                        column: 33,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchValue {
+                                                    value: Located {
+                                                        location: Location {
+                                                            row: 135,
+                                                            column: 32,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 135,
+                                                                column: 33,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Constant {
+                                                            value: Int(
+                                                                2,
+                                                            ),
+                                                            kind: None,
+                                                        },
+                                                    },
+                                                },
+                                            },
+                                        ),
+                                        name: Some(
+                                            "z",
+                                        ),
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: Some(
+                        Located {
+                            location: Location {
+                                row: 135,
+                                column: 43,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 135,
+                                    column: 53,
+                                },
+                            ),
+                            custom: (),
+                            node: Compare {
+                                left: Located {
+                                    location: Location {
+                                        row: 135,
+                                        column: 43,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 135,
+                                            column: 44,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "z",
+                                        ctx: Load,
+                                    },
+                                },
+                                ops: [
+                                    Eq,
+                                ],
+                                comparators: [
+                                    Located {
+                                        location: Location {
+                                            row: 135,
+                                            column: 48,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 135,
+                                                column: 53,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: BinOp {
+                                            left: Located {
+                                                location: Location {
+                                                    row: 135,
+                                                    column: 48,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 135,
+                                                        column: 49,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Name {
+                                                    id: "x",
+                                                    ctx: Load,
+                                                },
+                                            },
+                                            op: Mod,
+                                            right: Located {
+                                                location: Location {
+                                                    row: 135,
+                                                    column: 52,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 135,
+                                                        column: 53,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Constant {
+                                                    value: Int(
+                                                        2,
+                                                    ),
+                                                    kind: None,
+                                                },
+                                            },
+                                        },
+                                    },
+                                ],
+                            },
+                        },
+                    ),
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 136,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 136,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 136,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 136,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 136,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 136,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 138,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 146,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 138,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 138,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 139,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 139,
+                                column: 24,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchMapping {
+                            keys: [
+                                Located {
+                                    location: Location {
+                                        row: 139,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 139,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 139,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 139,
+                                            column: 23,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchSequence {
+                                        patterns: [
+                                            Located {
+                                                location: Location {
+                                                    row: 139,
+                                                    column: 14,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 139,
+                                                        column: 15,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchValue {
+                                                    value: Located {
+                                                        location: Location {
+                                                            row: 139,
+                                                            column: 14,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 139,
+                                                                column: 15,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Constant {
+                                                            value: Int(
+                                                                1,
+                                                            ),
+                                                            kind: None,
+                                                        },
+                                                    },
+                                                },
+                                            },
+                                            Located {
+                                                location: Location {
+                                                    row: 139,
+                                                    column: 17,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 139,
+                                                        column: 18,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchValue {
+                                                    value: Located {
+                                                        location: Location {
+                                                            row: 139,
+                                                            column: 17,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 139,
+                                                                column: 18,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Constant {
+                                                            value: Int(
+                                                                2,
+                                                            ),
+                                                            kind: None,
+                                                        },
+                                                    },
+                                                },
+                                            },
+                                            Located {
+                                                location: Location {
+                                                    row: 139,
+                                                    column: 20,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 139,
+                                                        column: 22,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchMapping {
+                                                    keys: [],
+                                                    patterns: [],
+                                                    rest: None,
+                                                },
+                                            },
+                                        ],
+                                    },
+                                },
+                            ],
+                            rest: None,
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 140,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 140,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 140,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 140,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 140,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 140,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 141,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 141,
+                                column: 78,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchOr {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 141,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 141,
+                                            column: 32,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchMapping {
+                                        keys: [
+                                            Located {
+                                                location: Location {
+                                                    row: 141,
+                                                    column: 10,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 141,
+                                                        column: 11,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Constant {
+                                                    value: Int(
+                                                        0,
+                                                    ),
+                                                    kind: None,
+                                                },
+                                            },
+                                        ],
+                                        patterns: [
+                                            Located {
+                                                location: Location {
+                                                    row: 141,
+                                                    column: 13,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 141,
+                                                        column: 31,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchOr {
+                                                    patterns: [
+                                                        Located {
+                                                            location: Location {
+                                                                row: 141,
+                                                                column: 13,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 141,
+                                                                    column: 23,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchSequence {
+                                                                patterns: [
+                                                                    Located {
+                                                                        location: Location {
+                                                                            row: 141,
+                                                                            column: 14,
+                                                                        },
+                                                                        end_location: Some(
+                                                                            Location {
+                                                                                row: 141,
+                                                                                column: 15,
+                                                                            },
+                                                                        ),
+                                                                        custom: (),
+                                                                        node: MatchValue {
+                                                                            value: Located {
+                                                                                location: Location {
+                                                                                    row: 141,
+                                                                                    column: 14,
+                                                                                },
+                                                                                end_location: Some(
+                                                                                    Location {
+                                                                                        row: 141,
+                                                                                        column: 15,
+                                                                                    },
+                                                                                ),
+                                                                                custom: (),
+                                                                                node: Constant {
+                                                                                    value: Int(
+                                                                                        1,
+                                                                                    ),
+                                                                                    kind: None,
+                                                                                },
+                                                                            },
+                                                                        },
+                                                                    },
+                                                                    Located {
+                                                                        location: Location {
+                                                                            row: 141,
+                                                                            column: 17,
+                                                                        },
+                                                                        end_location: Some(
+                                                                            Location {
+                                                                                row: 141,
+                                                                                column: 18,
+                                                                            },
+                                                                        ),
+                                                                        custom: (),
+                                                                        node: MatchValue {
+                                                                            value: Located {
+                                                                                location: Location {
+                                                                                    row: 141,
+                                                                                    column: 17,
+                                                                                },
+                                                                                end_location: Some(
+                                                                                    Location {
+                                                                                        row: 141,
+                                                                                        column: 18,
+                                                                                    },
+                                                                                ),
+                                                                                custom: (),
+                                                                                node: Constant {
+                                                                                    value: Int(
+                                                                                        2,
+                                                                                    ),
+                                                                                    kind: None,
+                                                                                },
+                                                                            },
+                                                                        },
+                                                                    },
+                                                                    Located {
+                                                                        location: Location {
+                                                                            row: 141,
+                                                                            column: 20,
+                                                                        },
+                                                                        end_location: Some(
+                                                                            Location {
+                                                                                row: 141,
+                                                                                column: 22,
+                                                                            },
+                                                                        ),
+                                                                        custom: (),
+                                                                        node: MatchMapping {
+                                                                            keys: [],
+                                                                            patterns: [],
+                                                                            rest: None,
+                                                                        },
+                                                                    },
+                                                                ],
+                                                            },
+                                                        },
+                                                        Located {
+                                                            location: Location {
+                                                                row: 141,
+                                                                column: 26,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 141,
+                                                                    column: 31,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchSingleton {
+                                                                value: Bool(
+                                                                    false,
+                                                                ),
+                                                            },
+                                                        },
+                                                    ],
+                                                },
+                                            },
+                                        ],
+                                        rest: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 141,
+                                        column: 35,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 141,
+                                            column: 44,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchMapping {
+                                        keys: [
+                                            Located {
+                                                location: Location {
+                                                    row: 141,
+                                                    column: 36,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 141,
+                                                        column: 37,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Constant {
+                                                    value: Int(
+                                                        1,
+                                                    ),
+                                                    kind: None,
+                                                },
+                                            },
+                                        ],
+                                        patterns: [
+                                            Located {
+                                                location: Location {
+                                                    row: 141,
+                                                    column: 39,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 141,
+                                                        column: 43,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchSequence {
+                                                    patterns: [
+                                                        Located {
+                                                            location: Location {
+                                                                row: 141,
+                                                                column: 40,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 141,
+                                                                    column: 42,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchSequence {
+                                                                patterns: [],
+                                                            },
+                                                        },
+                                                    ],
+                                                },
+                                            },
+                                        ],
+                                        rest: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 141,
+                                        column: 47,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 141,
+                                            column: 62,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchMapping {
+                                        keys: [
+                                            Located {
+                                                location: Location {
+                                                    row: 141,
+                                                    column: 48,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 141,
+                                                        column: 49,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Constant {
+                                                    value: Int(
+                                                        0,
+                                                    ),
+                                                    kind: None,
+                                                },
+                                            },
+                                        ],
+                                        patterns: [
+                                            Located {
+                                                location: Location {
+                                                    row: 141,
+                                                    column: 51,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 141,
+                                                        column: 61,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchSequence {
+                                                    patterns: [
+                                                        Located {
+                                                            location: Location {
+                                                                row: 141,
+                                                                column: 52,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 141,
+                                                                    column: 53,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchValue {
+                                                                value: Located {
+                                                                    location: Location {
+                                                                        row: 141,
+                                                                        column: 52,
+                                                                    },
+                                                                    end_location: Some(
+                                                                        Location {
+                                                                            row: 141,
+                                                                            column: 53,
+                                                                        },
+                                                                    ),
+                                                                    custom: (),
+                                                                    node: Constant {
+                                                                        value: Int(
+                                                                            1,
+                                                                        ),
+                                                                        kind: None,
+                                                                    },
+                                                                },
+                                                            },
+                                                        },
+                                                        Located {
+                                                            location: Location {
+                                                                row: 141,
+                                                                column: 55,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 141,
+                                                                    column: 56,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchValue {
+                                                                value: Located {
+                                                                    location: Location {
+                                                                        row: 141,
+                                                                        column: 55,
+                                                                    },
+                                                                    end_location: Some(
+                                                                        Location {
+                                                                            row: 141,
+                                                                            column: 56,
+                                                                        },
+                                                                    ),
+                                                                    custom: (),
+                                                                    node: Constant {
+                                                                        value: Int(
+                                                                            2,
+                                                                        ),
+                                                                        kind: None,
+                                                                    },
+                                                                },
+                                                            },
+                                                        },
+                                                        Located {
+                                                            location: Location {
+                                                                row: 141,
+                                                                column: 58,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 141,
+                                                                    column: 60,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchMapping {
+                                                                keys: [],
+                                                                patterns: [],
+                                                                rest: None,
+                                                            },
+                                                        },
+                                                    ],
+                                                },
+                                            },
+                                        ],
+                                        rest: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 141,
+                                        column: 65,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 141,
+                                            column: 67,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchSequence {
+                                        patterns: [],
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 141,
+                                        column: 70,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 141,
+                                            column: 73,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 141,
+                                                column: 70,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 141,
+                                                    column: 73,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Str(
+                                                    "X",
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 141,
+                                        column: 76,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 141,
+                                            column: 78,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchMapping {
+                                        keys: [],
+                                        patterns: [],
+                                        rest: None,
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 142,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 142,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 142,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 142,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 142,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 142,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            1,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 143,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 143,
+                                column: 11,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 144,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 144,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 144,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 144,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 144,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 144,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            2,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 146,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 150,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 146,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 146,
+                        column: 15,
+                    },
+                ),
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            location: Location {
+                                row: 146,
+                                column: 7,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 146,
+                                    column: 8,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    0,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 146,
+                                column: 10,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 146,
+                                    column: 11,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 146,
+                                column: 13,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 146,
+                                    column: 14,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 147,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 147,
+                                column: 14,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 147,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 147,
+                                            column: 10,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 147,
+                                                column: 9,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 147,
+                                                    column: 10,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 147,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 147,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchStar {
+                                        name: Some(
+                                            "x",
+                                        ),
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 148,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 148,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 148,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 148,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 148,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 148,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 150,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 154,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 150,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 150,
+                        column: 15,
+                    },
+                ),
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            location: Location {
+                                row: 150,
+                                column: 7,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 150,
+                                    column: 8,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    0,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 150,
+                                column: 10,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 150,
+                                    column: 11,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 150,
+                                column: 13,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 150,
+                                    column: 14,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    2,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 151,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 151,
+                                column: 15,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 151,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 151,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchStar {
+                                        name: Some(
+                                            "x",
+                                        ),
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 151,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 151,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 151,
+                                                column: 13,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 151,
+                                                    column: 14,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    2,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 152,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 152,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 152,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 152,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 152,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 152,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 154,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 158,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 154,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 154,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 155,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 155,
+                                column: 11,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 155,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 155,
+                                            column: 10,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: None,
+                                        name: Some(
+                                            "y",
+                                        ),
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 156,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 156,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 156,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 156,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "z",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 156,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 156,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 158,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 162,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 158,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 162,
+                        column: 0,
+                    },
+                ),
+                custom: (),
+                node: Tuple {
+                    elts: [
+                        Located {
+                            location: Location {
+                                row: 158,
+                                column: 6,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 158,
+                                    column: 7,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "w",
+                                ctx: Load,
+                            },
+                        },
+                        Located {
+                            location: Location {
+                                row: 158,
+                                column: 9,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 158,
+                                    column: 10,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "x",
+                                ctx: Load,
+                            },
+                        },
+                    ],
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 159,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 159,
+                                column: 13,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 159,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 159,
+                                            column: 10,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: None,
+                                        name: Some(
+                                            "y",
+                                        ),
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 159,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 159,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: None,
+                                        name: Some(
+                                            "z",
+                                        ),
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 160,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 160,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 160,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 160,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "v",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 160,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 160,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 162,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 165,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 162,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 162,
+                        column: 12,
+                    },
+                ),
+                custom: (),
+                node: NamedExpr {
+                    target: Located {
+                        location: Location {
+                            row: 162,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 162,
+                                column: 7,
+                            },
+                        ),
+                        custom: (),
+                        node: Name {
+                            id: "w",
+                            ctx: Store,
+                        },
+                    },
+                    value: Located {
+                        location: Location {
+                            row: 162,
+                            column: 11,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 162,
+                                column: 12,
+                            },
+                        ),
+                        custom: (),
+                        node: Name {
+                            id: "x",
+                            ctx: Load,
+                        },
+                    },
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 163,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 163,
+                                column: 16,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 163,
+                                        column: 9,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 163,
+                                            column: 15,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: Some(
+                                            Located {
+                                                location: Location {
+                                                    row: 163,
+                                                    column: 9,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 163,
+                                                        column: 10,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchAs {
+                                                    pattern: None,
+                                                    name: Some(
+                                                        "y",
+                                                    ),
+                                                },
+                                            },
+                                        ),
+                                        name: Some(
+                                            "v",
+                                        ),
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 164,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 164,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 164,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 164,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "z",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 164,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 164,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+]
diff --git a/parser/src/soft_keywords.rs b/parser/src/soft_keywords.rs
new file mode 100644
index 00000000..5a969297
--- /dev/null
+++ b/parser/src/soft_keywords.rs
@@ -0,0 +1,117 @@
+use crate::lexer::{LexResult, Tok};
+pub use crate::mode::Mode;
+
+/// Collect all tokens from a token stream in a vector.
+fn collect_tokens(tokenizer: impl IntoIterator<Item = LexResult>) -> Vec<LexResult> {
+    let mut tokens: Vec<LexResult> = vec![];
+    for tok in tokenizer {
+        let is_err = tok.is_err();
+        tokens.push(tok);
+        if is_err {
+            break;
+        }
+    }
+    tokens
+}
+
+/// Modify a token stream to accommodate soft keywords (namely, `match` and `case`).
+///
+/// [PEP 634](https://www.python.org/dev/peps/pep-0634/) introduced the `match` and `case` keywords
+/// as soft keywords, meaning that they can be used as identifiers (e.g., variable names) in certain
+/// contexts.
+///
+/// This function modifies a token stream to accommodate this change. In particular, it replaces
+/// `match` and `case` tokens with `identifier` tokens if they are used as identifiers.
+///
+/// Handling soft keywords in this intermediary pass allows us to simplify both the lexer and
+/// parser, as neither of them need to be aware of soft keywords.
+pub fn soft_keywords(
+    tokenizer: impl IntoIterator<Item = LexResult>,
+    mode: Mode,
+) -> Vec<LexResult> {
+    let mut tokenizer: Vec<LexResult> = collect_tokens(tokenizer);
+    let mut start_of_line = matches!(mode, Mode::Module | Mode::Interactive);
+    for i in 0..tokenizer.len() {
+        // If the token is a `match` or `case` token, check if it's used as an identifier.
+        // We assume every `match` or `case` is an identifier unless both of the following
+        // conditions are met:
+        // 1. The token is at the start of a logical line.
+        // 2. The logical line contains a top-level colon (that is, a colon that is not nested
+        //    inside a parenthesized expression, list, or dictionary).
+        // 3. The top-level colon is not the immediate sibling of a `match` or `case` token.
+        //    (This is to avoid treating `match` and `case` as identifiers when annotated with
+        //    type hints.)
+        if tokenizer[i]
+            .as_ref()
+            .map_or(false, |(_, tok, _)| matches!(tok, Tok::Match | Tok::Case))
+        {
+            let is_identifier = {
+                if !start_of_line {
+                    // If the `match` or `case` token is not at the start of a line, it's definitely
+                    // an identifier.
+                    true
+                } else {
+                    //
+                    let mut seen_colon = false;
+                    let mut first = true;
+                    let mut par_count = 0;
+                    let mut sqb_count = 0;
+                    let mut brace_count = 0;
+                    for (_, tok, _) in tokenizer.iter().skip(i + 1).flatten() {
+                        match tok {
+                            Tok::Newline => break,
+                            Tok::Colon if par_count == 0 && sqb_count == 0 && brace_count == 0 => {
+                                if !first {
+                                    seen_colon = true;
+                                }
+                                break;
+                            }
+                            Tok::Lpar => {
+                                par_count += 1;
+                            }
+                            Tok::Rpar => {
+                                par_count -= 1;
+                            }
+                            Tok::Lsqb => {
+                                sqb_count += 1;
+                            }
+                            Tok::Rsqb => {
+                                sqb_count -= 1;
+                            }
+                            Tok::Lbrace => {
+                                brace_count += 1;
+                            }
+                            Tok::Rbrace => {
+                                brace_count -= 1;
+                            }
+                            _ => {}
+                        }
+                        first = false;
+                    }
+                    !seen_colon
+                }
+            };
+            if is_identifier {
+                if let Ok((_, tok, _)) = &mut tokenizer[i] {
+                    if let Tok::Match = tok {
+                        *tok = Tok::Name {
+                            name: "match".to_string(),
+                        };
+                    } else if let Tok::Case = tok {
+                        *tok = Tok::Name {
+                            name: "case".to_string(),
+                        };
+                    }
+                }
+            }
+        }
+        start_of_line = tokenizer[i].as_ref().map_or(false, |(_, tok, _)| {
+            matches!(
+                tok,
+                Tok::StartModule | Tok::StartInteractive | Tok::Newline | Tok::Indent | Tok::Dedent
+            )
+        });
+    }
+
+    tokenizer
+}
diff --git a/parser/src/token.rs b/parser/src/token.rs
index 3fa6ff14..42b09915 100644
--- a/parser/src/token.rs
+++ b/parser/src/token.rs
@@ -184,6 +184,8 @@ pub enum Tok {
     Return,
     Try,
     While,
+    Match,
+    Case,
     With,
     Yield,
 
@@ -297,6 +299,8 @@ impl fmt::Display for Tok {
             Return => f.write_str("'return'"),
             Try => f.write_str("'try'"),
             While => f.write_str("'while'"),
+            Match => f.write_str("'match'"),
+            Case => f.write_str("'case'"),
             With => f.write_str("'with'"),
             Yield => f.write_str("'yield'"),
             ColonEqual => f.write_str("':='"),

From f1f31324d03204fc74321f433b51a4a246360884 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Sun, 19 Feb 2023 22:37:00 -0500
Subject: [PATCH 419/459] Use muiltipeek

---
 parser/src/parser.rs        |   4 +-
 parser/src/soft_keywords.rs | 173 +++++++++++++++++++-----------------
 2 files changed, 93 insertions(+), 84 deletions(-)

diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index d8f09cd4..f4c00f0f 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -14,11 +14,11 @@
 
 use crate::lexer::{LexResult, Tok};
 pub use crate::mode::Mode;
+use crate::soft_keywords::SoftKeywordTransformer;
 use crate::{ast, error::ParseError, lexer, python};
 use ast::Location;
 use itertools::Itertools;
 use std::iter;
-use crate::soft_keywords::soft_keywords;
 
 /// Parse a full Python program usually consisting of multiple lines.
 ///  
@@ -190,7 +190,7 @@ pub fn parse_tokens(
         .chain(lxr)
         .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
     python::TopParser::new()
-        .parse(soft_keywords(tokenizer, mode).into_iter())
+        .parse(SoftKeywordTransformer::new(tokenizer, mode).into_iter())
         .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
 }
 
diff --git a/parser/src/soft_keywords.rs b/parser/src/soft_keywords.rs
index 5a969297..b2c16f92 100644
--- a/parser/src/soft_keywords.rs
+++ b/parser/src/soft_keywords.rs
@@ -1,20 +1,10 @@
+use itertools::{Itertools, MultiPeek};
+
 use crate::lexer::{LexResult, Tok};
 pub use crate::mode::Mode;
 
-/// Collect all tokens from a token stream in a vector.
-fn collect_tokens(tokenizer: impl IntoIterator<Item = LexResult>) -> Vec<LexResult> {
-    let mut tokens: Vec<LexResult> = vec![];
-    for tok in tokenizer {
-        let is_err = tok.is_err();
-        tokens.push(tok);
-        if is_err {
-            break;
-        }
-    }
-    tokens
-}
-
-/// Modify a token stream to accommodate soft keywords (namely, `match` and `case`).
+/// An [`Iterator`] that transforms a token stream to accommodate soft keywords (namely, `match`
+/// and `case`).
 ///
 /// [PEP 634](https://www.python.org/dev/peps/pep-0634/) introduced the `match` and `case` keywords
 /// as soft keywords, meaning that they can be used as identifiers (e.g., variable names) in certain
@@ -25,93 +15,112 @@ fn collect_tokens(tokenizer: impl IntoIterator<Item = LexResult>) -> Vec<LexResu
 ///
 /// Handling soft keywords in this intermediary pass allows us to simplify both the lexer and
 /// parser, as neither of them need to be aware of soft keywords.
-pub fn soft_keywords(
-    tokenizer: impl IntoIterator<Item = LexResult>,
-    mode: Mode,
-) -> Vec<LexResult> {
-    let mut tokenizer: Vec<LexResult> = collect_tokens(tokenizer);
-    let mut start_of_line = matches!(mode, Mode::Module | Mode::Interactive);
-    for i in 0..tokenizer.len() {
-        // If the token is a `match` or `case` token, check if it's used as an identifier.
-        // We assume every `match` or `case` is an identifier unless both of the following
-        // conditions are met:
-        // 1. The token is at the start of a logical line.
-        // 2. The logical line contains a top-level colon (that is, a colon that is not nested
-        //    inside a parenthesized expression, list, or dictionary).
-        // 3. The top-level colon is not the immediate sibling of a `match` or `case` token.
-        //    (This is to avoid treating `match` and `case` as identifiers when annotated with
-        //    type hints.)
-        if tokenizer[i]
-            .as_ref()
-            .map_or(false, |(_, tok, _)| matches!(tok, Tok::Match | Tok::Case))
-        {
-            let is_identifier = {
-                if !start_of_line {
-                    // If the `match` or `case` token is not at the start of a line, it's definitely
-                    // an identifier.
-                    true
+pub struct SoftKeywordTransformer<I>
+where
+    I: Iterator<Item = LexResult>,
+{
+    pub underlying: MultiPeek<I>,
+    pub start_of_line: bool,
+}
+
+impl<I> SoftKeywordTransformer<I>
+where
+    I: Iterator<Item = LexResult>,
+{
+    pub fn new(tokenizer: I, mode: Mode) -> Self {
+        Self {
+            underlying: tokenizer.multipeek(),
+            start_of_line: matches!(mode, Mode::Interactive | Mode::Module),
+        }
+    }
+}
+
+impl<I> Iterator for SoftKeywordTransformer<I>
+where
+    I: Iterator<Item = LexResult>,
+{
+    type Item = LexResult;
+
+    #[inline]
+    fn next(&mut self) -> Option<LexResult> {
+        let mut next = self.underlying.next();
+        if let Some(Ok((start, tok, end))) = next.as_ref() {
+            // If the token is a `match` or `case` token, check if it's used as an identifier.
+            // We assume every `match` or `case` is an identifier unless both of the following
+            // conditions are met:
+            // 1. The token is at the start of a logical line.
+            // 2. The logical line contains a top-level colon (that is, a colon that is not nested
+            //    inside a parenthesized expression, list, or dictionary).
+            // 3. The top-level colon is not the immediate sibling of a `match` or `case` token.
+            //    (This is to avoid treating `match` and `case` as identifiers when annotated with
+            //    type hints.)
+            if matches!(tok, Tok::Match | Tok::Case) {
+                if !self.start_of_line {
+                    next = Some(Ok((
+                        *start,
+                        Tok::Name {
+                            name: if matches!(tok, Tok::Match) {
+                                "match".to_string()
+                            } else {
+                                "case".to_string()
+                            },
+                        },
+                        *end,
+                    )));
                 } else {
-                    //
-                    let mut seen_colon = false;
-                    let mut first = true;
                     let mut par_count = 0;
                     let mut sqb_count = 0;
                     let mut brace_count = 0;
-                    for (_, tok, _) in tokenizer.iter().skip(i + 1).flatten() {
+                    let mut first = true;
+                    let mut seen_colon = false;
+                    while let Some(Ok((_, tok, _))) = self.underlying.peek() {
                         match tok {
                             Tok::Newline => break,
                             Tok::Colon if par_count == 0 && sqb_count == 0 && brace_count == 0 => {
                                 if !first {
                                     seen_colon = true;
                                 }
-                                break;
-                            }
-                            Tok::Lpar => {
-                                par_count += 1;
-                            }
-                            Tok::Rpar => {
-                                par_count -= 1;
-                            }
-                            Tok::Lsqb => {
-                                sqb_count += 1;
-                            }
-                            Tok::Rsqb => {
-                                sqb_count -= 1;
-                            }
-                            Tok::Lbrace => {
-                                brace_count += 1;
-                            }
-                            Tok::Rbrace => {
-                                brace_count -= 1;
                             }
+                            Tok::Lpar => par_count += 1,
+                            Tok::Rpar => par_count -= 1,
+                            Tok::Lsqb => sqb_count += 1,
+                            Tok::Rsqb => sqb_count -= 1,
+                            Tok::Lbrace => brace_count += 1,
+                            Tok::Rbrace => brace_count -= 1,
                             _ => {}
                         }
                         first = false;
                     }
-                    !seen_colon
-                }
-            };
-            if is_identifier {
-                if let Ok((_, tok, _)) = &mut tokenizer[i] {
-                    if let Tok::Match = tok {
-                        *tok = Tok::Name {
-                            name: "match".to_string(),
-                        };
-                    } else if let Tok::Case = tok {
-                        *tok = Tok::Name {
-                            name: "case".to_string(),
-                        };
+                    if !seen_colon {
+                        next = Some(Ok((
+                            *start,
+                            Tok::Name {
+                                name: if matches!(tok, Tok::Match) {
+                                    "match".to_string()
+                                } else {
+                                    "case".to_string()
+                                },
+                            },
+                            *end,
+                        )));
                     }
                 }
             }
         }
-        start_of_line = tokenizer[i].as_ref().map_or(false, |(_, tok, _)| {
-            matches!(
-                tok,
-                Tok::StartModule | Tok::StartInteractive | Tok::Newline | Tok::Indent | Tok::Dedent
-            )
+
+        self.start_of_line = next.as_ref().map_or(false, |lex_result| {
+            lex_result.as_ref().map_or(false, |(_, tok, _)| {
+                matches!(
+                    tok,
+                    Tok::StartModule
+                        | Tok::StartInteractive
+                        | Tok::Newline
+                        | Tok::Indent
+                        | Tok::Dedent
+                )
+            })
         });
-    }
 
-    tokenizer
+        next
+    }
 }

From 591036785c470da54e004eaf88ed9701ef56c7e3 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Mon, 20 Feb 2023 14:14:08 +0900
Subject: [PATCH 420/459] clean up soft-keyword transform

---
 parser/src/soft_keywords.rs | 36 ++++++++++++++----------------------
 1 file changed, 14 insertions(+), 22 deletions(-)

diff --git a/parser/src/soft_keywords.rs b/parser/src/soft_keywords.rs
index b2c16f92..d5eaa298 100644
--- a/parser/src/soft_keywords.rs
+++ b/parser/src/soft_keywords.rs
@@ -56,17 +56,7 @@ where
             //    type hints.)
             if matches!(tok, Tok::Match | Tok::Case) {
                 if !self.start_of_line {
-                    next = Some(Ok((
-                        *start,
-                        Tok::Name {
-                            name: if matches!(tok, Tok::Match) {
-                                "match".to_string()
-                            } else {
-                                "case".to_string()
-                            },
-                        },
-                        *end,
-                    )));
+                    next = Some(Ok((*start, soft_to_name(tok), *end)));
                 } else {
                     let mut par_count = 0;
                     let mut sqb_count = 0;
@@ -92,17 +82,7 @@ where
                         first = false;
                     }
                     if !seen_colon {
-                        next = Some(Ok((
-                            *start,
-                            Tok::Name {
-                                name: if matches!(tok, Tok::Match) {
-                                    "match".to_string()
-                                } else {
-                                    "case".to_string()
-                                },
-                            },
-                            *end,
-                        )));
+                        next = Some(Ok((*start, soft_to_name(tok), *end)));
                     }
                 }
             }
@@ -124,3 +104,15 @@ where
         next
     }
 }
+
+#[inline]
+fn soft_to_name(tok: &Tok) -> Tok {
+    let name = match tok {
+        Tok::Match => "match",
+        Tok::Case => "case",
+        _ => unreachable!("other tokens never reach here"),
+    };
+    Tok::Name {
+        name: name.to_owned(),
+    }
+}

From 77e827acd5a32dc83382b3e0fe04ef2e151bfd35 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <crmarsh416@gmail.com>
Date: Mon, 20 Feb 2023 13:20:26 -0500
Subject: [PATCH 421/459] Update compiler/parser/src/soft_keywords.rs

Co-authored-by: Jim Fasarakis-Hilliard <d.f.hilliard@gmail.com>
---
 parser/src/soft_keywords.rs | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/parser/src/soft_keywords.rs b/parser/src/soft_keywords.rs
index d5eaa298..a4256252 100644
--- a/parser/src/soft_keywords.rs
+++ b/parser/src/soft_keywords.rs
@@ -58,25 +58,19 @@ where
                 if !self.start_of_line {
                     next = Some(Ok((*start, soft_to_name(tok), *end)));
                 } else {
-                    let mut par_count = 0;
-                    let mut sqb_count = 0;
-                    let mut brace_count = 0;
+                    let mut nesting = 0;
                     let mut first = true;
                     let mut seen_colon = false;
                     while let Some(Ok((_, tok, _))) = self.underlying.peek() {
                         match tok {
                             Tok::Newline => break,
-                            Tok::Colon if par_count == 0 && sqb_count == 0 && brace_count == 0 => {
+                            Tok::Colon if nesting == 0 => {
                                 if !first {
                                     seen_colon = true;
                                 }
                             }
-                            Tok::Lpar => par_count += 1,
-                            Tok::Rpar => par_count -= 1,
-                            Tok::Lsqb => sqb_count += 1,
-                            Tok::Rsqb => sqb_count -= 1,
-                            Tok::Lbrace => brace_count += 1,
-                            Tok::Rbrace => brace_count -= 1,
+                            Tok::Lpar | Tok::Lsqb | Tok::Lbrace => nesting += 1,
+                            Tok::Rpar | Tok::Rsqb | Tok::Rbrace => nesting -= 1,
                             _ => {}
                         }
                         first = false;

From 4c559a9a514bdf73d2eed12ee3afe85df44badb5 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Mon, 20 Feb 2023 17:59:35 -0500
Subject: [PATCH 422/459] Allow starred expressions in subscripts

---
 parser/python.lalrpop                         |   2 +-
 parser/src/parser.rs                          |  12 +
 ...hon_parser__parser__tests__star_index.snap | 641 ++++++++++++++++++
 3 files changed, 654 insertions(+), 1 deletion(-)
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__star_index.snap

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 78b71361..10c19f41 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -1037,7 +1037,7 @@ SubscriptList: ast::Expr = {
 };
 
 Subscript: ast::Expr = {
-    NamedExpressionTest,
+    TestOrStarNamedExpr,
     <location:@L> <e1:Test<"all">?> ":" <e2:Test<"all">?> <e3:SliceOp?> <end_location:@R> => {
         let lower = e1.map(Box::new);
         let upper = e2.map(Box::new);
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 7f828021..0ae4d104 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -392,6 +392,18 @@ with (0 as a, 1 as b,): pass
         }
     }
 
+    #[test]
+    fn test_star_index() {
+        let source = "\
+array_slice = array[0, *idxs, -1]
+array[0, *idxs, -1] = array_slice
+array[*idxs_to_select, *idxs_to_select]
+array[3:5, *idxs_to_select]
+";
+        let parse_ast = parse_program(source, "<test>").unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
     #[test]
     fn test_generator_expression_argument() {
         let source = r#"' '.join(
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__star_index.snap b/parser/src/snapshots/rustpython_parser__parser__tests__star_index.snap
new file mode 100644
index 00000000..6aed23b8
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__star_index.snap
@@ -0,0 +1,641 @@
+---
+source: compiler/parser/src/parser.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 33,
+            },
+        ),
+        custom: (),
+        node: Assign {
+            targets: [
+                Located {
+                    location: Location {
+                        row: 1,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 1,
+                            column: 11,
+                        },
+                    ),
+                    custom: (),
+                    node: Name {
+                        id: "array_slice",
+                        ctx: Store,
+                    },
+                },
+            ],
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 14,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 33,
+                    },
+                ),
+                custom: (),
+                node: Subscript {
+                    value: Located {
+                        location: Location {
+                            row: 1,
+                            column: 14,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 19,
+                            },
+                        ),
+                        custom: (),
+                        node: Name {
+                            id: "array",
+                            ctx: Load,
+                        },
+                    },
+                    slice: Located {
+                        location: Location {
+                            row: 1,
+                            column: 20,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 1,
+                                column: 32,
+                            },
+                        ),
+                        custom: (),
+                        node: Tuple {
+                            elts: [
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 20,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 21,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 23,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 28,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Starred {
+                                        value: Located {
+                                            location: Location {
+                                                row: 1,
+                                                column: 24,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 1,
+                                                    column: 28,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "idxs",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        ctx: Load,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 1,
+                                        column: 30,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 1,
+                                            column: 32,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: UnaryOp {
+                                        op: USub,
+                                        operand: Located {
+                                            location: Location {
+                                                row: 1,
+                                                column: 31,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 1,
+                                                    column: 32,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    1,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                            ctx: Load,
+                        },
+                    },
+                    ctx: Load,
+                },
+            },
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 2,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 2,
+                column: 33,
+            },
+        ),
+        custom: (),
+        node: Assign {
+            targets: [
+                Located {
+                    location: Location {
+                        row: 2,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 2,
+                            column: 19,
+                        },
+                    ),
+                    custom: (),
+                    node: Subscript {
+                        value: Located {
+                            location: Location {
+                                row: 2,
+                                column: 0,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 5,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "array",
+                                ctx: Load,
+                            },
+                        },
+                        slice: Located {
+                            location: Location {
+                                row: 2,
+                                column: 6,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 18,
+                                },
+                            ),
+                            custom: (),
+                            node: Tuple {
+                                elts: [
+                                    Located {
+                                        location: Location {
+                                            row: 2,
+                                            column: 6,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 2,
+                                                column: 7,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Constant {
+                                            value: Int(
+                                                0,
+                                            ),
+                                            kind: None,
+                                        },
+                                    },
+                                    Located {
+                                        location: Location {
+                                            row: 2,
+                                            column: 9,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 2,
+                                                column: 14,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Starred {
+                                            value: Located {
+                                                location: Location {
+                                                    row: 2,
+                                                    column: 10,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 2,
+                                                        column: 14,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Name {
+                                                    id: "idxs",
+                                                    ctx: Load,
+                                                },
+                                            },
+                                            ctx: Load,
+                                        },
+                                    },
+                                    Located {
+                                        location: Location {
+                                            row: 2,
+                                            column: 16,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 2,
+                                                column: 18,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: UnaryOp {
+                                            op: USub,
+                                            operand: Located {
+                                                location: Location {
+                                                    row: 2,
+                                                    column: 17,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 2,
+                                                        column: 18,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Constant {
+                                                    value: Int(
+                                                        1,
+                                                    ),
+                                                    kind: None,
+                                                },
+                                            },
+                                        },
+                                    },
+                                ],
+                                ctx: Load,
+                            },
+                        },
+                        ctx: Store,
+                    },
+                },
+            ],
+            value: Located {
+                location: Location {
+                    row: 2,
+                    column: 22,
+                },
+                end_location: Some(
+                    Location {
+                        row: 2,
+                        column: 33,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "array_slice",
+                    ctx: Load,
+                },
+            },
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 3,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 3,
+                column: 39,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 3,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 3,
+                        column: 39,
+                    },
+                ),
+                custom: (),
+                node: Subscript {
+                    value: Located {
+                        location: Location {
+                            row: 3,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 3,
+                                column: 5,
+                            },
+                        ),
+                        custom: (),
+                        node: Name {
+                            id: "array",
+                            ctx: Load,
+                        },
+                    },
+                    slice: Located {
+                        location: Location {
+                            row: 3,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 3,
+                                column: 38,
+                            },
+                        ),
+                        custom: (),
+                        node: Tuple {
+                            elts: [
+                                Located {
+                                    location: Location {
+                                        row: 3,
+                                        column: 6,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 3,
+                                            column: 21,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Starred {
+                                        value: Located {
+                                            location: Location {
+                                                row: 3,
+                                                column: 7,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 3,
+                                                    column: 21,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "idxs_to_select",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        ctx: Load,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 3,
+                                        column: 23,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 3,
+                                            column: 38,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Starred {
+                                        value: Located {
+                                            location: Location {
+                                                row: 3,
+                                                column: 24,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 3,
+                                                    column: 38,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "idxs_to_select",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        ctx: Load,
+                                    },
+                                },
+                            ],
+                            ctx: Load,
+                        },
+                    },
+                    ctx: Load,
+                },
+            },
+        },
+    },
+    Located {
+        location: Location {
+            row: 4,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 4,
+                column: 27,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 4,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 4,
+                        column: 27,
+                    },
+                ),
+                custom: (),
+                node: Subscript {
+                    value: Located {
+                        location: Location {
+                            row: 4,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 4,
+                                column: 5,
+                            },
+                        ),
+                        custom: (),
+                        node: Name {
+                            id: "array",
+                            ctx: Load,
+                        },
+                    },
+                    slice: Located {
+                        location: Location {
+                            row: 4,
+                            column: 6,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 4,
+                                column: 26,
+                            },
+                        ),
+                        custom: (),
+                        node: Tuple {
+                            elts: [
+                                Located {
+                                    location: Location {
+                                        row: 4,
+                                        column: 6,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 4,
+                                            column: 9,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Slice {
+                                        lower: Some(
+                                            Located {
+                                                location: Location {
+                                                    row: 4,
+                                                    column: 6,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 4,
+                                                        column: 7,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Constant {
+                                                    value: Int(
+                                                        3,
+                                                    ),
+                                                    kind: None,
+                                                },
+                                            },
+                                        ),
+                                        upper: Some(
+                                            Located {
+                                                location: Location {
+                                                    row: 4,
+                                                    column: 8,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 4,
+                                                        column: 9,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Constant {
+                                                    value: Int(
+                                                        5,
+                                                    ),
+                                                    kind: None,
+                                                },
+                                            },
+                                        ),
+                                        step: None,
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 4,
+                                        column: 11,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 4,
+                                            column: 26,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Starred {
+                                        value: Located {
+                                            location: Location {
+                                                row: 4,
+                                                column: 12,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 4,
+                                                    column: 26,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "idxs_to_select",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        ctx: Load,
+                                    },
+                                },
+                            ],
+                            ctx: Load,
+                        },
+                    },
+                    ctx: Load,
+                },
+            },
+        },
+    },
+]

From fab9adcd4846096de641643eb4d23b47b7cbd6f0 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Tue, 21 Feb 2023 12:00:33 +0900
Subject: [PATCH 423/459] Update Python.asdl from CPython 3.11.1

---
 ast/Python.asdl    |  1 +
 ast/src/ast_gen.rs | 17 +++++++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/ast/Python.asdl b/ast/Python.asdl
index 32fdc01a..e9423a7c 100644
--- a/ast/Python.asdl
+++ b/ast/Python.asdl
@@ -40,6 +40,7 @@ module Python
 
           | Raise(expr? exc, expr? cause)
           | Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)
+          | TryStar(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)
           | Assert(expr test, expr? msg)
 
           | Import(alias* names)
diff --git a/ast/src/ast_gen.rs b/ast/src/ast_gen.rs
index 32dd89c2..c11fd87d 100644
--- a/ast/src/ast_gen.rs
+++ b/ast/src/ast_gen.rs
@@ -139,6 +139,12 @@ pub enum StmtKind<U = ()> {
         orelse: Vec<Stmt<U>>,
         finalbody: Vec<Stmt<U>>,
     },
+    TryStar {
+        body: Vec<Stmt<U>>,
+        handlers: Vec<Excepthandler<U>>,
+        orelse: Vec<Stmt<U>>,
+        finalbody: Vec<Stmt<U>>,
+    },
     Assert {
         test: Box<Expr<U>>,
         msg: Option<Box<Expr<U>>>,
@@ -726,6 +732,17 @@ pub mod fold {
                 orelse: Foldable::fold(orelse, folder)?,
                 finalbody: Foldable::fold(finalbody, folder)?,
             }),
+            StmtKind::TryStar {
+                body,
+                handlers,
+                orelse,
+                finalbody,
+            } => Ok(StmtKind::TryStar {
+                body: Foldable::fold(body, folder)?,
+                handlers: Foldable::fold(handlers, folder)?,
+                orelse: Foldable::fold(orelse, folder)?,
+                finalbody: Foldable::fold(finalbody, folder)?,
+            }),
             StmtKind::Assert { test, msg } => Ok(StmtKind::Assert {
                 test: Foldable::fold(test, folder)?,
                 msg: Foldable::fold(msg, folder)?,

From 2a86dbd6446d006db5c05a3686b478869dc38195 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Mon, 20 Feb 2023 13:40:19 -0500
Subject: [PATCH 424/459] Implement except* syntax

---
 codegen/src/compile.rs                        |  16 +
 codegen/src/symboltable.rs                    |   6 +
 parser/python.lalrpop                         |  49 +
 parser/src/parser.rs                          |  31 +
 ...rustpython_parser__parser__tests__try.snap | 487 ++++++++++
 ...ython_parser__parser__tests__try_star.snap | 861 ++++++++++++++++++
 6 files changed, 1450 insertions(+)
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__try.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__try_star.snap

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index c64cfb89..e3c17bcd 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -709,6 +709,12 @@ impl Compiler {
                 orelse,
                 finalbody,
             } => self.compile_try_statement(body, handlers, orelse, finalbody)?,
+            TryStar {
+                body,
+                handlers,
+                orelse,
+                finalbody,
+            } => self.compile_try_star_statement(body, handlers, orelse, finalbody)?,
             FunctionDef {
                 name,
                 args,
@@ -1092,6 +1098,16 @@ impl Compiler {
         Ok(())
     }
 
+    fn compile_try_star_statement(
+        &mut self,
+        _body: &[ast::Stmt],
+        _handlers: &[ast::Excepthandler],
+        _orelse: &[ast::Stmt],
+        _finalbody: &[ast::Stmt],
+    ) -> CompileResult<()> {
+        Err(self.error(CodegenErrorType::NotImplementedYet))
+    }
+
     fn is_forbidden_arg_name(name: &str) -> bool {
         is_forbidden_name(name)
     }
diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index dd2d2954..f2a7c42b 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -809,6 +809,12 @@ impl SymbolTableBuilder {
                 handlers,
                 orelse,
                 finalbody,
+            }
+            | TryStar {
+                body,
+                handlers,
+                orelse,
+                finalbody,
             } => {
                 self.scan_statements(body)?;
                 for handler in handlers {
diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 78b71361..d37f93e6 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -451,6 +451,27 @@ TryStatement: ast::Stmt = {
             },
         }
     },
+    <location:@L> "try" ":" <body:Suite> <handlers:ExceptStarClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> <end_location:@R> => {
+        let orelse = else_suite.map(|s| s.2).unwrap_or_default();
+        let finalbody = finally.map(|s| s.2).unwrap_or_default();
+        let end_location = finalbody
+            .last()
+            .or_else(|| orelse.last())
+            .map(|last| last.end_location)
+            .or_else(|| handlers.last().map(|last| last.end_location))
+            .unwrap();
+        ast::Stmt {
+            custom: (),
+            location,
+            end_location,
+            node: ast::StmtKind::TryStar {
+                body,
+                handlers,
+                orelse,
+                finalbody,
+            },
+        }
+    },
     <location:@L> "try" ":" <body:Suite> <finally:("finally" ":" Suite)> => {
         let handlers = vec![];
         let orelse = vec![];
@@ -470,6 +491,34 @@ TryStatement: ast::Stmt = {
     },
 };
 
+ExceptStarClause: ast::Excepthandler = {
+    <location:@L> "except" "*" <typ:Test<"all">> ":" <body:Suite> => {
+        let end_location = body.last().unwrap().end_location.unwrap();
+        ast::Excepthandler::new(
+            location,
+            end_location,
+            ast::ExcepthandlerKind::ExceptHandler {
+                type_: Some(Box::new(typ)),
+                name: None,
+                body,
+            },
+        )
+    },
+    <location:@L> "except" "*" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
+        let end_location = body.last().unwrap().end_location.unwrap();
+        ast::Excepthandler::new(
+            location,
+            end_location,
+            ast::ExcepthandlerKind::ExceptHandler {
+                type_: Some(Box::new(x.0)),
+                name: Some(x.2),
+                body,
+            },
+        )
+    },
+};
+
+
 ExceptClause: ast::Excepthandler = {
     <location:@L> "except" <typ:Test<"all">?> ":" <body:Suite> => {
         let end_location = body.last().unwrap().end_location.unwrap();
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 7f828021..cc3cb18c 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -405,6 +405,37 @@ with (0 as a, 1 as b,): pass
         insta::assert_debug_snapshot!(parse_ast);
     }
 
+    #[test]
+    fn test_try() {
+        let parse_ast = parse_program(
+            r#"try:
+    raise ValueError(1)
+except TypeError as e:
+    print(f'caught {type(e)}')
+except OSError as e:
+    print(f'caught {type(e)}')"#,
+            "<test>",
+        )
+        .unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_try_star() {
+        let parse_ast = parse_program(
+            r#"try:
+    raise ExceptionGroup("eg",
+        [ValueError(1), TypeError(2), OSError(3), OSError(4)])
+except* TypeError as e:
+    print(f'caught {type(e)} with nested {e.exceptions}')
+except* OSError as e:
+    print(f'caught {type(e)} with nested {e.exceptions}')"#,
+            "<test>",
+        )
+        .unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
     #[test]
     fn test_dict_unpacking() {
         let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "<test>").unwrap();
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__try.snap b/parser/src/snapshots/rustpython_parser__parser__tests__try.snap
new file mode 100644
index 00000000..5b6037ae
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__try.snap
@@ -0,0 +1,487 @@
+---
+source: compiler/parser/src/parser.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 6,
+                column: 30,
+            },
+        ),
+        custom: (),
+        node: Try {
+            body: [
+                Located {
+                    location: Location {
+                        row: 2,
+                        column: 4,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 2,
+                            column: 23,
+                        },
+                    ),
+                    custom: (),
+                    node: Raise {
+                        exc: Some(
+                            Located {
+                                location: Location {
+                                    row: 2,
+                                    column: 10,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 2,
+                                        column: 23,
+                                    },
+                                ),
+                                custom: (),
+                                node: Call {
+                                    func: Located {
+                                        location: Location {
+                                            row: 2,
+                                            column: 10,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 2,
+                                                column: 20,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "ValueError",
+                                            ctx: Load,
+                                        },
+                                    },
+                                    args: [
+                                        Located {
+                                            location: Location {
+                                                row: 2,
+                                                column: 21,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 2,
+                                                    column: 22,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    1,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    ],
+                                    keywords: [],
+                                },
+                            },
+                        ),
+                        cause: None,
+                    },
+                },
+            ],
+            handlers: [
+                Located {
+                    location: Location {
+                        row: 3,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 4,
+                            column: 30,
+                        },
+                    ),
+                    custom: (),
+                    node: ExceptHandler {
+                        type_: Some(
+                            Located {
+                                location: Location {
+                                    row: 3,
+                                    column: 7,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 3,
+                                        column: 16,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "TypeError",
+                                    ctx: Load,
+                                },
+                            },
+                        ),
+                        name: Some(
+                            "e",
+                        ),
+                        body: [
+                            Located {
+                                location: Location {
+                                    row: 4,
+                                    column: 4,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 4,
+                                        column: 30,
+                                    },
+                                ),
+                                custom: (),
+                                node: Expr {
+                                    value: Located {
+                                        location: Location {
+                                            row: 4,
+                                            column: 4,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 4,
+                                                column: 30,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Call {
+                                            func: Located {
+                                                location: Location {
+                                                    row: 4,
+                                                    column: 4,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 4,
+                                                        column: 9,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Name {
+                                                    id: "print",
+                                                    ctx: Load,
+                                                },
+                                            },
+                                            args: [
+                                                Located {
+                                                    location: Location {
+                                                        row: 4,
+                                                        column: 10,
+                                                    },
+                                                    end_location: Some(
+                                                        Location {
+                                                            row: 4,
+                                                            column: 29,
+                                                        },
+                                                    ),
+                                                    custom: (),
+                                                    node: JoinedStr {
+                                                        values: [
+                                                            Located {
+                                                                location: Location {
+                                                                    row: 4,
+                                                                    column: 10,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 4,
+                                                                        column: 29,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: Constant {
+                                                                    value: Str(
+                                                                        "caught ",
+                                                                    ),
+                                                                    kind: None,
+                                                                },
+                                                            },
+                                                            Located {
+                                                                location: Location {
+                                                                    row: 4,
+                                                                    column: 10,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 4,
+                                                                        column: 29,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: FormattedValue {
+                                                                    value: Located {
+                                                                        location: Location {
+                                                                            row: 4,
+                                                                            column: 20,
+                                                                        },
+                                                                        end_location: Some(
+                                                                            Location {
+                                                                                row: 4,
+                                                                                column: 27,
+                                                                            },
+                                                                        ),
+                                                                        custom: (),
+                                                                        node: Call {
+                                                                            func: Located {
+                                                                                location: Location {
+                                                                                    row: 4,
+                                                                                    column: 20,
+                                                                                },
+                                                                                end_location: Some(
+                                                                                    Location {
+                                                                                        row: 4,
+                                                                                        column: 24,
+                                                                                    },
+                                                                                ),
+                                                                                custom: (),
+                                                                                node: Name {
+                                                                                    id: "type",
+                                                                                    ctx: Load,
+                                                                                },
+                                                                            },
+                                                                            args: [
+                                                                                Located {
+                                                                                    location: Location {
+                                                                                        row: 4,
+                                                                                        column: 25,
+                                                                                    },
+                                                                                    end_location: Some(
+                                                                                        Location {
+                                                                                            row: 4,
+                                                                                            column: 26,
+                                                                                        },
+                                                                                    ),
+                                                                                    custom: (),
+                                                                                    node: Name {
+                                                                                        id: "e",
+                                                                                        ctx: Load,
+                                                                                    },
+                                                                                },
+                                                                            ],
+                                                                            keywords: [],
+                                                                        },
+                                                                    },
+                                                                    conversion: 0,
+                                                                    format_spec: None,
+                                                                },
+                                                            },
+                                                        ],
+                                                    },
+                                                },
+                                            ],
+                                            keywords: [],
+                                        },
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+                Located {
+                    location: Location {
+                        row: 5,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 6,
+                            column: 30,
+                        },
+                    ),
+                    custom: (),
+                    node: ExceptHandler {
+                        type_: Some(
+                            Located {
+                                location: Location {
+                                    row: 5,
+                                    column: 7,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 5,
+                                        column: 14,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "OSError",
+                                    ctx: Load,
+                                },
+                            },
+                        ),
+                        name: Some(
+                            "e",
+                        ),
+                        body: [
+                            Located {
+                                location: Location {
+                                    row: 6,
+                                    column: 4,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 6,
+                                        column: 30,
+                                    },
+                                ),
+                                custom: (),
+                                node: Expr {
+                                    value: Located {
+                                        location: Location {
+                                            row: 6,
+                                            column: 4,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 6,
+                                                column: 30,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Call {
+                                            func: Located {
+                                                location: Location {
+                                                    row: 6,
+                                                    column: 4,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 6,
+                                                        column: 9,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Name {
+                                                    id: "print",
+                                                    ctx: Load,
+                                                },
+                                            },
+                                            args: [
+                                                Located {
+                                                    location: Location {
+                                                        row: 6,
+                                                        column: 10,
+                                                    },
+                                                    end_location: Some(
+                                                        Location {
+                                                            row: 6,
+                                                            column: 29,
+                                                        },
+                                                    ),
+                                                    custom: (),
+                                                    node: JoinedStr {
+                                                        values: [
+                                                            Located {
+                                                                location: Location {
+                                                                    row: 6,
+                                                                    column: 10,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 6,
+                                                                        column: 29,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: Constant {
+                                                                    value: Str(
+                                                                        "caught ",
+                                                                    ),
+                                                                    kind: None,
+                                                                },
+                                                            },
+                                                            Located {
+                                                                location: Location {
+                                                                    row: 6,
+                                                                    column: 10,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 6,
+                                                                        column: 29,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: FormattedValue {
+                                                                    value: Located {
+                                                                        location: Location {
+                                                                            row: 6,
+                                                                            column: 20,
+                                                                        },
+                                                                        end_location: Some(
+                                                                            Location {
+                                                                                row: 6,
+                                                                                column: 27,
+                                                                            },
+                                                                        ),
+                                                                        custom: (),
+                                                                        node: Call {
+                                                                            func: Located {
+                                                                                location: Location {
+                                                                                    row: 6,
+                                                                                    column: 20,
+                                                                                },
+                                                                                end_location: Some(
+                                                                                    Location {
+                                                                                        row: 6,
+                                                                                        column: 24,
+                                                                                    },
+                                                                                ),
+                                                                                custom: (),
+                                                                                node: Name {
+                                                                                    id: "type",
+                                                                                    ctx: Load,
+                                                                                },
+                                                                            },
+                                                                            args: [
+                                                                                Located {
+                                                                                    location: Location {
+                                                                                        row: 6,
+                                                                                        column: 25,
+                                                                                    },
+                                                                                    end_location: Some(
+                                                                                        Location {
+                                                                                            row: 6,
+                                                                                            column: 26,
+                                                                                        },
+                                                                                    ),
+                                                                                    custom: (),
+                                                                                    node: Name {
+                                                                                        id: "e",
+                                                                                        ctx: Load,
+                                                                                    },
+                                                                                },
+                                                                            ],
+                                                                            keywords: [],
+                                                                        },
+                                                                    },
+                                                                    conversion: 0,
+                                                                    format_spec: None,
+                                                                },
+                                                            },
+                                                        ],
+                                                    },
+                                                },
+                                            ],
+                                            keywords: [],
+                                        },
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            ],
+            orelse: [],
+            finalbody: [],
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__try_star.snap b/parser/src/snapshots/rustpython_parser__parser__tests__try_star.snap
new file mode 100644
index 00000000..472f316b
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__try_star.snap
@@ -0,0 +1,861 @@
+---
+source: compiler/parser/src/parser.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 7,
+                column: 57,
+            },
+        ),
+        custom: (),
+        node: TryStar {
+            body: [
+                Located {
+                    location: Location {
+                        row: 2,
+                        column: 4,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 3,
+                            column: 62,
+                        },
+                    ),
+                    custom: (),
+                    node: Raise {
+                        exc: Some(
+                            Located {
+                                location: Location {
+                                    row: 2,
+                                    column: 10,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 3,
+                                        column: 62,
+                                    },
+                                ),
+                                custom: (),
+                                node: Call {
+                                    func: Located {
+                                        location: Location {
+                                            row: 2,
+                                            column: 10,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 2,
+                                                column: 24,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "ExceptionGroup",
+                                            ctx: Load,
+                                        },
+                                    },
+                                    args: [
+                                        Located {
+                                            location: Location {
+                                                row: 2,
+                                                column: 25,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 2,
+                                                    column: 29,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Str(
+                                                    "eg",
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                        Located {
+                                            location: Location {
+                                                row: 3,
+                                                column: 8,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 3,
+                                                    column: 61,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: List {
+                                                elts: [
+                                                    Located {
+                                                        location: Location {
+                                                            row: 3,
+                                                            column: 9,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 3,
+                                                                column: 22,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Call {
+                                                            func: Located {
+                                                                location: Location {
+                                                                    row: 3,
+                                                                    column: 9,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 3,
+                                                                        column: 19,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: Name {
+                                                                    id: "ValueError",
+                                                                    ctx: Load,
+                                                                },
+                                                            },
+                                                            args: [
+                                                                Located {
+                                                                    location: Location {
+                                                                        row: 3,
+                                                                        column: 20,
+                                                                    },
+                                                                    end_location: Some(
+                                                                        Location {
+                                                                            row: 3,
+                                                                            column: 21,
+                                                                        },
+                                                                    ),
+                                                                    custom: (),
+                                                                    node: Constant {
+                                                                        value: Int(
+                                                                            1,
+                                                                        ),
+                                                                        kind: None,
+                                                                    },
+                                                                },
+                                                            ],
+                                                            keywords: [],
+                                                        },
+                                                    },
+                                                    Located {
+                                                        location: Location {
+                                                            row: 3,
+                                                            column: 24,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 3,
+                                                                column: 36,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Call {
+                                                            func: Located {
+                                                                location: Location {
+                                                                    row: 3,
+                                                                    column: 24,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 3,
+                                                                        column: 33,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: Name {
+                                                                    id: "TypeError",
+                                                                    ctx: Load,
+                                                                },
+                                                            },
+                                                            args: [
+                                                                Located {
+                                                                    location: Location {
+                                                                        row: 3,
+                                                                        column: 34,
+                                                                    },
+                                                                    end_location: Some(
+                                                                        Location {
+                                                                            row: 3,
+                                                                            column: 35,
+                                                                        },
+                                                                    ),
+                                                                    custom: (),
+                                                                    node: Constant {
+                                                                        value: Int(
+                                                                            2,
+                                                                        ),
+                                                                        kind: None,
+                                                                    },
+                                                                },
+                                                            ],
+                                                            keywords: [],
+                                                        },
+                                                    },
+                                                    Located {
+                                                        location: Location {
+                                                            row: 3,
+                                                            column: 38,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 3,
+                                                                column: 48,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Call {
+                                                            func: Located {
+                                                                location: Location {
+                                                                    row: 3,
+                                                                    column: 38,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 3,
+                                                                        column: 45,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: Name {
+                                                                    id: "OSError",
+                                                                    ctx: Load,
+                                                                },
+                                                            },
+                                                            args: [
+                                                                Located {
+                                                                    location: Location {
+                                                                        row: 3,
+                                                                        column: 46,
+                                                                    },
+                                                                    end_location: Some(
+                                                                        Location {
+                                                                            row: 3,
+                                                                            column: 47,
+                                                                        },
+                                                                    ),
+                                                                    custom: (),
+                                                                    node: Constant {
+                                                                        value: Int(
+                                                                            3,
+                                                                        ),
+                                                                        kind: None,
+                                                                    },
+                                                                },
+                                                            ],
+                                                            keywords: [],
+                                                        },
+                                                    },
+                                                    Located {
+                                                        location: Location {
+                                                            row: 3,
+                                                            column: 50,
+                                                        },
+                                                        end_location: Some(
+                                                            Location {
+                                                                row: 3,
+                                                                column: 60,
+                                                            },
+                                                        ),
+                                                        custom: (),
+                                                        node: Call {
+                                                            func: Located {
+                                                                location: Location {
+                                                                    row: 3,
+                                                                    column: 50,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 3,
+                                                                        column: 57,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: Name {
+                                                                    id: "OSError",
+                                                                    ctx: Load,
+                                                                },
+                                                            },
+                                                            args: [
+                                                                Located {
+                                                                    location: Location {
+                                                                        row: 3,
+                                                                        column: 58,
+                                                                    },
+                                                                    end_location: Some(
+                                                                        Location {
+                                                                            row: 3,
+                                                                            column: 59,
+                                                                        },
+                                                                    ),
+                                                                    custom: (),
+                                                                    node: Constant {
+                                                                        value: Int(
+                                                                            4,
+                                                                        ),
+                                                                        kind: None,
+                                                                    },
+                                                                },
+                                                            ],
+                                                            keywords: [],
+                                                        },
+                                                    },
+                                                ],
+                                                ctx: Load,
+                                            },
+                                        },
+                                    ],
+                                    keywords: [],
+                                },
+                            },
+                        ),
+                        cause: None,
+                    },
+                },
+            ],
+            handlers: [
+                Located {
+                    location: Location {
+                        row: 4,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 5,
+                            column: 57,
+                        },
+                    ),
+                    custom: (),
+                    node: ExceptHandler {
+                        type_: Some(
+                            Located {
+                                location: Location {
+                                    row: 4,
+                                    column: 8,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 4,
+                                        column: 17,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "TypeError",
+                                    ctx: Load,
+                                },
+                            },
+                        ),
+                        name: Some(
+                            "e",
+                        ),
+                        body: [
+                            Located {
+                                location: Location {
+                                    row: 5,
+                                    column: 4,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 5,
+                                        column: 57,
+                                    },
+                                ),
+                                custom: (),
+                                node: Expr {
+                                    value: Located {
+                                        location: Location {
+                                            row: 5,
+                                            column: 4,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 5,
+                                                column: 57,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Call {
+                                            func: Located {
+                                                location: Location {
+                                                    row: 5,
+                                                    column: 4,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 5,
+                                                        column: 9,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Name {
+                                                    id: "print",
+                                                    ctx: Load,
+                                                },
+                                            },
+                                            args: [
+                                                Located {
+                                                    location: Location {
+                                                        row: 5,
+                                                        column: 10,
+                                                    },
+                                                    end_location: Some(
+                                                        Location {
+                                                            row: 5,
+                                                            column: 56,
+                                                        },
+                                                    ),
+                                                    custom: (),
+                                                    node: JoinedStr {
+                                                        values: [
+                                                            Located {
+                                                                location: Location {
+                                                                    row: 5,
+                                                                    column: 10,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 5,
+                                                                        column: 56,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: Constant {
+                                                                    value: Str(
+                                                                        "caught ",
+                                                                    ),
+                                                                    kind: None,
+                                                                },
+                                                            },
+                                                            Located {
+                                                                location: Location {
+                                                                    row: 5,
+                                                                    column: 10,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 5,
+                                                                        column: 56,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: FormattedValue {
+                                                                    value: Located {
+                                                                        location: Location {
+                                                                            row: 5,
+                                                                            column: 20,
+                                                                        },
+                                                                        end_location: Some(
+                                                                            Location {
+                                                                                row: 5,
+                                                                                column: 27,
+                                                                            },
+                                                                        ),
+                                                                        custom: (),
+                                                                        node: Call {
+                                                                            func: Located {
+                                                                                location: Location {
+                                                                                    row: 5,
+                                                                                    column: 20,
+                                                                                },
+                                                                                end_location: Some(
+                                                                                    Location {
+                                                                                        row: 5,
+                                                                                        column: 24,
+                                                                                    },
+                                                                                ),
+                                                                                custom: (),
+                                                                                node: Name {
+                                                                                    id: "type",
+                                                                                    ctx: Load,
+                                                                                },
+                                                                            },
+                                                                            args: [
+                                                                                Located {
+                                                                                    location: Location {
+                                                                                        row: 5,
+                                                                                        column: 25,
+                                                                                    },
+                                                                                    end_location: Some(
+                                                                                        Location {
+                                                                                            row: 5,
+                                                                                            column: 26,
+                                                                                        },
+                                                                                    ),
+                                                                                    custom: (),
+                                                                                    node: Name {
+                                                                                        id: "e",
+                                                                                        ctx: Load,
+                                                                                    },
+                                                                                },
+                                                                            ],
+                                                                            keywords: [],
+                                                                        },
+                                                                    },
+                                                                    conversion: 0,
+                                                                    format_spec: None,
+                                                                },
+                                                            },
+                                                            Located {
+                                                                location: Location {
+                                                                    row: 5,
+                                                                    column: 10,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 5,
+                                                                        column: 56,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: Constant {
+                                                                    value: Str(
+                                                                        " with nested ",
+                                                                    ),
+                                                                    kind: None,
+                                                                },
+                                                            },
+                                                            Located {
+                                                                location: Location {
+                                                                    row: 5,
+                                                                    column: 10,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 5,
+                                                                        column: 56,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: FormattedValue {
+                                                                    value: Located {
+                                                                        location: Location {
+                                                                            row: 5,
+                                                                            column: 42,
+                                                                        },
+                                                                        end_location: Some(
+                                                                            Location {
+                                                                                row: 5,
+                                                                                column: 54,
+                                                                            },
+                                                                        ),
+                                                                        custom: (),
+                                                                        node: Attribute {
+                                                                            value: Located {
+                                                                                location: Location {
+                                                                                    row: 5,
+                                                                                    column: 42,
+                                                                                },
+                                                                                end_location: Some(
+                                                                                    Location {
+                                                                                        row: 5,
+                                                                                        column: 43,
+                                                                                    },
+                                                                                ),
+                                                                                custom: (),
+                                                                                node: Name {
+                                                                                    id: "e",
+                                                                                    ctx: Load,
+                                                                                },
+                                                                            },
+                                                                            attr: "exceptions",
+                                                                            ctx: Load,
+                                                                        },
+                                                                    },
+                                                                    conversion: 0,
+                                                                    format_spec: None,
+                                                                },
+                                                            },
+                                                        ],
+                                                    },
+                                                },
+                                            ],
+                                            keywords: [],
+                                        },
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+                Located {
+                    location: Location {
+                        row: 6,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 7,
+                            column: 57,
+                        },
+                    ),
+                    custom: (),
+                    node: ExceptHandler {
+                        type_: Some(
+                            Located {
+                                location: Location {
+                                    row: 6,
+                                    column: 8,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 6,
+                                        column: 15,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "OSError",
+                                    ctx: Load,
+                                },
+                            },
+                        ),
+                        name: Some(
+                            "e",
+                        ),
+                        body: [
+                            Located {
+                                location: Location {
+                                    row: 7,
+                                    column: 4,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 7,
+                                        column: 57,
+                                    },
+                                ),
+                                custom: (),
+                                node: Expr {
+                                    value: Located {
+                                        location: Location {
+                                            row: 7,
+                                            column: 4,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 7,
+                                                column: 57,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Call {
+                                            func: Located {
+                                                location: Location {
+                                                    row: 7,
+                                                    column: 4,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 7,
+                                                        column: 9,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Name {
+                                                    id: "print",
+                                                    ctx: Load,
+                                                },
+                                            },
+                                            args: [
+                                                Located {
+                                                    location: Location {
+                                                        row: 7,
+                                                        column: 10,
+                                                    },
+                                                    end_location: Some(
+                                                        Location {
+                                                            row: 7,
+                                                            column: 56,
+                                                        },
+                                                    ),
+                                                    custom: (),
+                                                    node: JoinedStr {
+                                                        values: [
+                                                            Located {
+                                                                location: Location {
+                                                                    row: 7,
+                                                                    column: 10,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 7,
+                                                                        column: 56,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: Constant {
+                                                                    value: Str(
+                                                                        "caught ",
+                                                                    ),
+                                                                    kind: None,
+                                                                },
+                                                            },
+                                                            Located {
+                                                                location: Location {
+                                                                    row: 7,
+                                                                    column: 10,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 7,
+                                                                        column: 56,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: FormattedValue {
+                                                                    value: Located {
+                                                                        location: Location {
+                                                                            row: 7,
+                                                                            column: 20,
+                                                                        },
+                                                                        end_location: Some(
+                                                                            Location {
+                                                                                row: 7,
+                                                                                column: 27,
+                                                                            },
+                                                                        ),
+                                                                        custom: (),
+                                                                        node: Call {
+                                                                            func: Located {
+                                                                                location: Location {
+                                                                                    row: 7,
+                                                                                    column: 20,
+                                                                                },
+                                                                                end_location: Some(
+                                                                                    Location {
+                                                                                        row: 7,
+                                                                                        column: 24,
+                                                                                    },
+                                                                                ),
+                                                                                custom: (),
+                                                                                node: Name {
+                                                                                    id: "type",
+                                                                                    ctx: Load,
+                                                                                },
+                                                                            },
+                                                                            args: [
+                                                                                Located {
+                                                                                    location: Location {
+                                                                                        row: 7,
+                                                                                        column: 25,
+                                                                                    },
+                                                                                    end_location: Some(
+                                                                                        Location {
+                                                                                            row: 7,
+                                                                                            column: 26,
+                                                                                        },
+                                                                                    ),
+                                                                                    custom: (),
+                                                                                    node: Name {
+                                                                                        id: "e",
+                                                                                        ctx: Load,
+                                                                                    },
+                                                                                },
+                                                                            ],
+                                                                            keywords: [],
+                                                                        },
+                                                                    },
+                                                                    conversion: 0,
+                                                                    format_spec: None,
+                                                                },
+                                                            },
+                                                            Located {
+                                                                location: Location {
+                                                                    row: 7,
+                                                                    column: 10,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 7,
+                                                                        column: 56,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: Constant {
+                                                                    value: Str(
+                                                                        " with nested ",
+                                                                    ),
+                                                                    kind: None,
+                                                                },
+                                                            },
+                                                            Located {
+                                                                location: Location {
+                                                                    row: 7,
+                                                                    column: 10,
+                                                                },
+                                                                end_location: Some(
+                                                                    Location {
+                                                                        row: 7,
+                                                                        column: 56,
+                                                                    },
+                                                                ),
+                                                                custom: (),
+                                                                node: FormattedValue {
+                                                                    value: Located {
+                                                                        location: Location {
+                                                                            row: 7,
+                                                                            column: 42,
+                                                                        },
+                                                                        end_location: Some(
+                                                                            Location {
+                                                                                row: 7,
+                                                                                column: 54,
+                                                                            },
+                                                                        ),
+                                                                        custom: (),
+                                                                        node: Attribute {
+                                                                            value: Located {
+                                                                                location: Location {
+                                                                                    row: 7,
+                                                                                    column: 42,
+                                                                                },
+                                                                                end_location: Some(
+                                                                                    Location {
+                                                                                        row: 7,
+                                                                                        column: 43,
+                                                                                    },
+                                                                                ),
+                                                                                custom: (),
+                                                                                node: Name {
+                                                                                    id: "e",
+                                                                                    ctx: Load,
+                                                                                },
+                                                                            },
+                                                                            attr: "exceptions",
+                                                                            ctx: Load,
+                                                                        },
+                                                                    },
+                                                                    conversion: 0,
+                                                                    format_spec: None,
+                                                                },
+                                                            },
+                                                        ],
+                                                    },
+                                                },
+                                            ],
+                                            keywords: [],
+                                        },
+                                    },
+                                },
+                            },
+                        ],
+                    },
+                },
+            ],
+            orelse: [],
+            finalbody: [],
+        },
+    },
+]

From 82b91fe9aaabfcd95dd4ebfef38886828a93382b Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Tue, 21 Feb 2023 19:00:32 -0500
Subject: [PATCH 425/459] Expose SoftKeywordTransformer on public API

---
 parser/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index 4715dd96..9cc1dbca 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -131,5 +131,5 @@ mod string;
 #[rustfmt::skip]
 mod python;
 mod context;
-mod soft_keywords;
+pub mod soft_keywords;
 pub mod token;

From 62d88e0b8676fceea13dae5007749f60f41fded2 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Tue, 21 Feb 2023 19:18:42 -0500
Subject: [PATCH 426/459] Always wrap in SoftKeywordTransformer

---
 parser/src/lexer.rs         | 18 ++++++++++++------
 parser/src/lib.rs           | 10 ++++++----
 parser/src/parser.rs        | 19 +++++++++++--------
 parser/src/soft_keywords.rs | 24 ++++++++++++++----------
 4 files changed, 43 insertions(+), 28 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 20e6b148..2ea7867f 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -13,10 +13,11 @@
 //!
 //! ```
 //! use rustpython_parser::lexer::{make_tokenizer, Tok};
+//! use rustpython_parser::mode::Mode;
 //! use rustpython_parser::token::StringKind;
 //!
 //! let source = "x = 'RustPython'";
-//! let tokens = make_tokenizer(source)
+//! let tokens = make_tokenizer(source, Mode::Module)
 //!     .map(|tok| tok.expect("Failed to lex"))
 //!     .collect::<Vec<_>>();
 //!
@@ -35,6 +36,8 @@
 pub use super::token::{StringKind, Tok};
 use crate::ast::Location;
 use crate::error::{LexicalError, LexicalErrorType};
+use crate::mode::Mode;
+use crate::soft_keywords::SoftKeywordTransformer;
 use num_bigint::BigInt;
 use num_traits::identities::Zero;
 use num_traits::Num;
@@ -197,27 +200,29 @@ pub type LexResult = Result<Spanned, LexicalError>;
 /// # Examples
 ///
 /// ```
+/// use rustpython_parser::mode::Mode;
 /// use rustpython_parser::lexer::{make_tokenizer};
 ///
 /// let source = "def hello(): return 'world'";
-/// let tokenizer = make_tokenizer(source);
+/// let tokenizer = make_tokenizer(source, Mode::Module);
 ///
 /// for token in tokenizer {
 ///    println!("{:?}", token);
 /// }
 /// ```
 #[inline]
-pub fn make_tokenizer(source: &str) -> impl Iterator<Item = LexResult> + '_ {
-    make_tokenizer_located(source, Location::default())
+pub fn make_tokenizer(source: &str, mode: Mode) -> impl Iterator<Item = LexResult> + '_ {
+    make_tokenizer_located(source, mode, Location::default())
 }
 
 /// Create a new tokenizer from a source string, starting at a given location.
 /// You probably want to use [`make_tokenizer`] instead.
 pub fn make_tokenizer_located(
     source: &str,
+    mode: Mode,
     start_location: Location,
 ) -> impl Iterator<Item = LexResult> + '_ {
-    Lexer::new(source.chars(), start_location)
+    SoftKeywordTransformer::new(Lexer::new(source.chars(), start_location), mode)
 }
 
 impl<T> Lexer<T>
@@ -1210,6 +1215,7 @@ where
 #[cfg(test)]
 mod tests {
     use super::{make_tokenizer, StringKind, Tok};
+    use crate::mode::Mode;
     use num_bigint::BigInt;
 
     const WINDOWS_EOL: &str = "\r\n";
@@ -1217,7 +1223,7 @@ mod tests {
     const UNIX_EOL: &str = "\n";
 
     pub fn lex_source(source: &str) -> Vec<Tok> {
-        let lexer = make_tokenizer(source);
+        let lexer = make_tokenizer(source, Mode::Module);
         lexer.map(|x| x.unwrap().1).collect()
     }
 
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index 9cc1dbca..bfb56d74 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -66,27 +66,29 @@
 //! For example, to get a stream of tokens from a given string, one could do this:
 //!
 //! ```
+//! use rustpython_parser::mode::Mode;
 //! use rustpython_parser::lexer::make_tokenizer;
 //!
 //! let python_source = r#"
 //! def is_odd(i):
 //!     return bool(i & 1)
 //! "#;
-//! let mut tokens = make_tokenizer(python_source);
+//! let mut tokens = make_tokenizer(python_source, Mode::Module);
 //! assert!(tokens.all(|t| t.is_ok()));
 //! ```
 //!
 //! These tokens can be directly fed into the parser to generate an AST:
 //!
 //! ```
-//! use rustpython_parser::parser::{parse_tokens, Mode};
 //! use rustpython_parser::lexer::make_tokenizer;
+//! use rustpython_parser::mode::Mode;
+//! use rustpython_parser::parser::parse_tokens;
 //!
 //! let python_source = r#"
 //! def is_odd(i):
 //!    return bool(i & 1)
 //! "#;
-//! let tokens = make_tokenizer(python_source);
+//! let tokens = make_tokenizer(python_source, Mode::Module);
 //! let ast = parse_tokens(tokens, Mode::Module, "<embedded>");
 //!
 //! assert!(ast.is_ok());
@@ -131,5 +133,5 @@ mod string;
 #[rustfmt::skip]
 mod python;
 mod context;
-pub mod soft_keywords;
+mod soft_keywords;
 pub mod token;
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 65f9ea4b..e0865b2d 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -14,7 +14,6 @@
 
 use crate::lexer::{LexResult, Tok};
 pub use crate::mode::Mode;
-use crate::soft_keywords::SoftKeywordTransformer;
 use crate::{ast, error::ParseError, lexer, python};
 use ast::Location;
 use itertools::Itertools;
@@ -107,7 +106,8 @@ pub fn parse_expression_located(
 /// parsing:
 ///
 /// ```
-/// use rustpython_parser::parser::{parse, Mode};
+/// use rustpython_parser::mode::Mode;
+/// use rustpython_parser::parser::parse;
 ///
 /// let expr = parse("1 + 2", Mode::Expression, "<embedded>");
 /// assert!(expr.is_ok());
@@ -116,7 +116,8 @@ pub fn parse_expression_located(
 /// Alternatively, we can parse a full Python program consisting of multiple lines:
 ///
 /// ```
-/// use rustpython_parser::parser::{parse, Mode};
+/// use rustpython_parser::mode::Mode;
+/// use rustpython_parser::parser::parse;
 ///
 /// let source = r#"
 /// class Greeter:
@@ -139,8 +140,9 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
 /// # Example
 ///
 /// ```
-/// use rustpython_parser::parser::{parse_located, Mode};
 /// use rustpython_parser::ast::Location;
+/// use rustpython_parser::mode::Mode;
+/// use rustpython_parser::parser::parse_located;
 ///
 /// let source = r#"
 /// def fib(i):
@@ -160,7 +162,7 @@ pub fn parse_located(
     source_path: &str,
     location: Location,
 ) -> Result<ast::Mod, ParseError> {
-    let lxr = lexer::make_tokenizer_located(source, location);
+    let lxr = lexer::make_tokenizer_located(source, mode, location);
     parse_tokens(lxr, mode, source_path)
 }
 
@@ -174,10 +176,11 @@ pub fn parse_located(
 /// them using the [`lexer::make_tokenizer`] function:
 ///
 /// ```
-/// use rustpython_parser::parser::{parse_tokens, Mode};
 /// use rustpython_parser::lexer::make_tokenizer;
+/// use rustpython_parser::mode::Mode;
+/// use rustpython_parser::parser::parse_tokens;
 ///
-/// let expr = parse_tokens(make_tokenizer("1 + 2"), Mode::Expression, "<embedded>");
+/// let expr = parse_tokens(make_tokenizer("1 + 2", Mode::Expression), Mode::Expression, "<embedded>");
 /// assert!(expr.is_ok());
 /// ```
 pub fn parse_tokens(
@@ -190,7 +193,7 @@ pub fn parse_tokens(
         .chain(lxr)
         .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
     python::TopParser::new()
-        .parse(SoftKeywordTransformer::new(tokenizer, mode).into_iter())
+        .parse(tokenizer.into_iter())
         .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
 }
 
diff --git a/parser/src/soft_keywords.rs b/parser/src/soft_keywords.rs
index a4256252..a029ccd4 100644
--- a/parser/src/soft_keywords.rs
+++ b/parser/src/soft_keywords.rs
@@ -19,8 +19,8 @@ pub struct SoftKeywordTransformer<I>
 where
     I: Iterator<Item = LexResult>,
 {
-    pub underlying: MultiPeek<I>,
-    pub start_of_line: bool,
+    underlying: MultiPeek<I>,
+    start_of_line: bool,
 }
 
 impl<I> SoftKeywordTransformer<I>
@@ -84,14 +84,18 @@ where
 
         self.start_of_line = next.as_ref().map_or(false, |lex_result| {
             lex_result.as_ref().map_or(false, |(_, tok, _)| {
-                matches!(
-                    tok,
-                    Tok::StartModule
-                        | Tok::StartInteractive
-                        | Tok::Newline
-                        | Tok::Indent
-                        | Tok::Dedent
-                )
+                if matches!(tok, Tok::NonLogicalNewline | Tok::Comment { .. }) {
+                    self.start_of_line
+                } else {
+                    matches!(
+                        tok,
+                        Tok::StartModule
+                            | Tok::StartInteractive
+                            | Tok::Newline
+                            | Tok::Indent
+                            | Tok::Dedent
+                    )
+                }
             })
         });
 

From aed03c5209093466055d4d51e833855a966985cf Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 22 Feb 2023 16:16:12 +0900
Subject: [PATCH 427/459] Break down rustpython_parser::error module

because it doesn't share any common errors but specific error for each sub module
---
 parser/python.lalrpop  |   2 +-
 parser/src/error.rs    | 331 -----------------------------------------
 parser/src/function.rs |   6 +-
 parser/src/lexer.rs    | 108 +++++++++++++-
 parser/src/lib.rs      |   1 -
 parser/src/parser.rs   | 125 +++++++++++++++-
 parser/src/string.rs   | 110 +++++++++++++-
 src/lib.rs             |   6 +-
 8 files changed, 341 insertions(+), 348 deletions(-)
 delete mode 100644 parser/src/error.rs

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 9b3d226a..57d5dbe0 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -5,7 +5,7 @@
 
 use crate::{
     ast,
-    error::{LexicalError, LexicalErrorType},
+    lexer::{LexicalError, LexicalErrorType},
     function::{ArgumentList, parse_args, parse_params, validate_arguments},
     lexer,
     context::set_context,
diff --git a/parser/src/error.rs b/parser/src/error.rs
deleted file mode 100644
index dab07f8c..00000000
--- a/parser/src/error.rs
+++ /dev/null
@@ -1,331 +0,0 @@
-//! Error types for the parser.
-//!
-//! These types are used to represent errors that occur during lexing and parsing and are
-//! returned by the `parse_*` functions in the [parser] module and the iterator in the
-//! [lexer] implementation.
-//!
-//! [parser]: crate::parser
-//! [lexer]: crate::lexer
-
-// Define internal parse error types.
-// The goal is to provide a matching and a safe error API, masking errors from LALR
-use crate::{ast::Location, token::Tok};
-use lalrpop_util::ParseError as LalrpopError;
-use std::fmt;
-
-/// Represents an error during lexing.
-#[derive(Debug, PartialEq)]
-pub struct LexicalError {
-    /// The type of error that occurred.
-    pub error: LexicalErrorType,
-    /// The location of the error.
-    pub location: Location,
-}
-
-impl LexicalError {
-    /// Creates a new `LexicalError` with the given error type and location.
-    pub fn new(error: LexicalErrorType, location: Location) -> Self {
-        Self { error, location }
-    }
-}
-
-/// Represents the different types of errors that can occur during lexing.
-#[derive(Debug, PartialEq)]
-pub enum LexicalErrorType {
-    // TODO: Can probably be removed, the places it is used seem to be able
-    // to use the `UnicodeError` variant instead.
-    #[doc(hidden)]
-    StringError,
-    // TODO: Should take a start/end position to report.
-    /// Decoding of a unicode escape sequence in a string literal failed.
-    UnicodeError,
-    /// The nesting of brackets/braces/parentheses is not balanced.
-    NestingError,
-    /// The indentation is not consistent.
-    IndentationError,
-    /// Inconsistent use of tabs and spaces.
-    TabError,
-    /// Encountered a tab after a space.
-    TabsAfterSpaces,
-    /// A non-default argument follows a default argument.
-    DefaultArgumentError,
-    /// A duplicate argument was found in a function definition.
-    DuplicateArgumentError(String),
-    /// A positional argument follows a keyword argument.
-    PositionalArgumentError,
-    /// An iterable argument unpacking `*args` follows keyword argument unpacking `**kwargs`.
-    UnpackedArgumentError,
-    /// A keyword argument was repeated.
-    DuplicateKeywordArgumentError(String),
-    /// An unrecognized token was encountered.
-    UnrecognizedToken { tok: char },
-    /// An f-string error containing the [`FStringErrorType`].
-    FStringError(FStringErrorType),
-    /// An unexpected character was encountered after a line continuation.
-    LineContinuationError,
-    /// An unexpected end of file was encountered.
-    Eof,
-    /// An unexpected error occurred.
-    OtherError(String),
-}
-
-impl fmt::Display for LexicalErrorType {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            LexicalErrorType::StringError => write!(f, "Got unexpected string"),
-            LexicalErrorType::FStringError(error) => write!(f, "f-string: {error}"),
-            LexicalErrorType::UnicodeError => write!(f, "Got unexpected unicode"),
-            LexicalErrorType::NestingError => write!(f, "Got unexpected nesting"),
-            LexicalErrorType::IndentationError => {
-                write!(f, "unindent does not match any outer indentation level")
-            }
-            LexicalErrorType::TabError => {
-                write!(f, "inconsistent use of tabs and spaces in indentation")
-            }
-            LexicalErrorType::TabsAfterSpaces => {
-                write!(f, "Tabs not allowed as part of indentation after spaces")
-            }
-            LexicalErrorType::DefaultArgumentError => {
-                write!(f, "non-default argument follows default argument")
-            }
-            LexicalErrorType::DuplicateArgumentError(arg_name) => {
-                write!(f, "duplicate argument '{arg_name}' in function definition")
-            }
-            LexicalErrorType::DuplicateKeywordArgumentError(arg_name) => {
-                write!(f, "keyword argument repeated: {arg_name}")
-            }
-            LexicalErrorType::PositionalArgumentError => {
-                write!(f, "positional argument follows keyword argument")
-            }
-            LexicalErrorType::UnpackedArgumentError => {
-                write!(
-                    f,
-                    "iterable argument unpacking follows keyword argument unpacking"
-                )
-            }
-            LexicalErrorType::UnrecognizedToken { tok } => {
-                write!(f, "Got unexpected token {tok}")
-            }
-            LexicalErrorType::LineContinuationError => {
-                write!(f, "unexpected character after line continuation character")
-            }
-            LexicalErrorType::Eof => write!(f, "unexpected EOF while parsing"),
-            LexicalErrorType::OtherError(msg) => write!(f, "{msg}"),
-        }
-    }
-}
-
-// TODO: consolidate these with ParseError
-/// An error that occurred during parsing of an f-string.
-#[derive(Debug, PartialEq)]
-pub struct FStringError {
-    /// The type of error that occurred.
-    pub error: FStringErrorType,
-    /// The location of the error.
-    pub location: Location,
-}
-
-impl FStringError {
-    /// Creates a new `FStringError` with the given error type and location.
-    pub fn new(error: FStringErrorType, location: Location) -> Self {
-        Self { error, location }
-    }
-}
-
-impl From<FStringError> for LexicalError {
-    fn from(err: FStringError) -> Self {
-        LexicalError {
-            error: LexicalErrorType::FStringError(err.error),
-            location: err.location,
-        }
-    }
-}
-
-/// Represents the different types of errors that can occur during parsing of an f-string.
-#[derive(Debug, PartialEq)]
-pub enum FStringErrorType {
-    /// Expected a right brace after an opened left brace.
-    UnclosedLbrace,
-    /// Expected a left brace after an ending right brace.
-    UnopenedRbrace,
-    /// Expected a right brace after a conversion flag.
-    ExpectedRbrace,
-    /// An error occurred while parsing an f-string expression.
-    InvalidExpression(Box<ParseErrorType>),
-    /// An invalid conversion flag was encountered.
-    InvalidConversionFlag,
-    /// An empty expression was encountered.
-    EmptyExpression,
-    /// An opening delimiter was not closed properly.
-    MismatchedDelimiter(char, char),
-    /// Too many nested expressions in an f-string.
-    ExpressionNestedTooDeeply,
-    /// The f-string expression cannot include the given character.
-    ExpressionCannotInclude(char),
-    /// A single right brace was encountered.
-    SingleRbrace,
-    /// A closing delimiter was not opened properly.
-    Unmatched(char),
-    // TODO: Test this case.
-    /// Unterminated string.
-    UnterminatedString,
-}
-
-impl fmt::Display for FStringErrorType {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            FStringErrorType::UnclosedLbrace => write!(f, "expecting '}}'"),
-            FStringErrorType::UnopenedRbrace => write!(f, "Unopened '}}'"),
-            FStringErrorType::ExpectedRbrace => write!(f, "Expected '}}' after conversion flag."),
-            FStringErrorType::InvalidExpression(error) => {
-                write!(f, "{error}")
-            }
-            FStringErrorType::InvalidConversionFlag => write!(f, "invalid conversion character"),
-            FStringErrorType::EmptyExpression => write!(f, "empty expression not allowed"),
-            FStringErrorType::MismatchedDelimiter(first, second) => write!(
-                f,
-                "closing parenthesis '{second}' does not match opening parenthesis '{first}'"
-            ),
-            FStringErrorType::SingleRbrace => write!(f, "single '}}' is not allowed"),
-            FStringErrorType::Unmatched(delim) => write!(f, "unmatched '{delim}'"),
-            FStringErrorType::ExpressionNestedTooDeeply => {
-                write!(f, "expressions nested too deeply")
-            }
-            FStringErrorType::UnterminatedString => {
-                write!(f, "unterminated string")
-            }
-            FStringErrorType::ExpressionCannotInclude(c) => {
-                if *c == '\\' {
-                    write!(f, "f-string expression part cannot include a backslash")
-                } else {
-                    write!(f, "f-string expression part cannot include '{c}'s")
-                }
-            }
-        }
-    }
-}
-
-impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
-    fn from(err: FStringError) -> Self {
-        lalrpop_util::ParseError::User {
-            error: LexicalError {
-                error: LexicalErrorType::FStringError(err.error),
-                location: err.location,
-            },
-        }
-    }
-}
-
-/// Represents an error during parsing.
-pub type ParseError = rustpython_compiler_core::BaseError<ParseErrorType>;
-
-/// Represents the different types of errors that can occur during parsing.
-#[derive(Debug, PartialEq, thiserror::Error)]
-pub enum ParseErrorType {
-    /// Parser encountered an unexpected end of input
-    Eof,
-    /// Parser encountered an extra token
-    ExtraToken(Tok),
-    /// Parser encountered an invalid token
-    InvalidToken,
-    /// Parser encountered an unexpected token
-    UnrecognizedToken(Tok, Option<String>),
-    // Maps to `User` type from `lalrpop-util`
-    /// Parser encountered an error during lexing.
-    Lexical(LexicalErrorType),
-}
-
-// Convert `lalrpop_util::ParseError` to our internal type
-pub(crate) fn parse_error_from_lalrpop(
-    err: LalrpopError<Location, Tok, LexicalError>,
-    source_path: &str,
-) -> ParseError {
-    let source_path = source_path.to_owned();
-    match err {
-        // TODO: Are there cases where this isn't an EOF?
-        LalrpopError::InvalidToken { location } => ParseError {
-            error: ParseErrorType::Eof,
-            location,
-            source_path,
-        },
-        LalrpopError::ExtraToken { token } => ParseError {
-            error: ParseErrorType::ExtraToken(token.1),
-            location: token.0,
-            source_path,
-        },
-        LalrpopError::User { error } => ParseError {
-            error: ParseErrorType::Lexical(error.error),
-            location: error.location,
-            source_path,
-        },
-        LalrpopError::UnrecognizedToken { token, expected } => {
-            // Hacky, but it's how CPython does it. See PyParser_AddToken,
-            // in particular "Only one possible expected token" comment.
-            let expected = (expected.len() == 1).then(|| expected[0].clone());
-            ParseError {
-                error: ParseErrorType::UnrecognizedToken(token.1, expected),
-                location: token.0.with_col_offset(1),
-                source_path,
-            }
-        }
-        LalrpopError::UnrecognizedEOF { location, expected } => {
-            // This could be an initial indentation error that we should ignore
-            let indent_error = expected == ["Indent"];
-            if indent_error {
-                ParseError {
-                    error: ParseErrorType::Lexical(LexicalErrorType::IndentationError),
-                    location,
-                    source_path,
-                }
-            } else {
-                ParseError {
-                    error: ParseErrorType::Eof,
-                    location,
-                    source_path,
-                }
-            }
-        }
-    }
-}
-
-impl fmt::Display for ParseErrorType {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match *self {
-            ParseErrorType::Eof => write!(f, "Got unexpected EOF"),
-            ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {tok:?}"),
-            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
-            ParseErrorType::UnrecognizedToken(ref tok, ref expected) => {
-                if *tok == Tok::Indent {
-                    write!(f, "unexpected indent")
-                } else if expected.as_deref() == Some("Indent") {
-                    write!(f, "expected an indented block")
-                } else {
-                    write!(f, "invalid syntax. Got unexpected token {tok}")
-                }
-            }
-            ParseErrorType::Lexical(ref error) => write!(f, "{error}"),
-        }
-    }
-}
-
-impl ParseErrorType {
-    /// Returns true if the error is an indentation error.
-    pub fn is_indentation_error(&self) -> bool {
-        match self {
-            ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true,
-            ParseErrorType::UnrecognizedToken(token, expected) => {
-                *token == Tok::Indent || expected.clone() == Some("Indent".to_owned())
-            }
-            _ => false,
-        }
-    }
-
-    /// Returns true if the error is a tab error.
-    pub fn is_tab_error(&self) -> bool {
-        matches!(
-            self,
-            ParseErrorType::Lexical(LexicalErrorType::TabError)
-                | ParseErrorType::Lexical(LexicalErrorType::TabsAfterSpaces)
-        )
-    }
-}
diff --git a/parser/src/function.rs b/parser/src/function.rs
index 35c3ad29..dc950396 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -1,7 +1,7 @@
 // Contains functions that perform validation and parsing of arguments and parameters.
 // Checks apply both to functions and to lambdas.
 use crate::ast;
-use crate::error::{LexicalError, LexicalErrorType};
+use crate::lexer::{LexicalError, LexicalErrorType};
 use rustc_hash::FxHashSet;
 
 pub(crate) struct ArgumentList {
@@ -149,8 +149,8 @@ fn is_starred(exp: &ast::Expr) -> bool {
 
 #[cfg(test)]
 mod tests {
-    use crate::error::{LexicalErrorType, ParseErrorType};
-    use crate::parser::parse_program;
+    use crate::lexer::LexicalErrorType;
+    use crate::parser::{parse_program, ParseErrorType};
 
     macro_rules! function_and_lambda {
         ($($name:ident: $code:expr,)*) => {
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 2ea7867f..7da3b8c5 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -35,9 +35,9 @@
 //! [Lexical analysis]: https://docs.python.org/3/reference/lexical_analysis.html
 pub use super::token::{StringKind, Tok};
 use crate::ast::Location;
-use crate::error::{LexicalError, LexicalErrorType};
 use crate::mode::Mode;
 use crate::soft_keywords::SoftKeywordTransformer;
+use crate::string::FStringErrorType;
 use num_bigint::BigInt;
 use num_traits::identities::Zero;
 use num_traits::Num;
@@ -1212,6 +1212,112 @@ where
     }
 }
 
+/// Represents an error that occur during lexing and are
+/// returned by the `parse_*` functions in the iterator in the
+/// [lexer] implementation.
+///
+/// [lexer]: crate::lexer
+#[derive(Debug, PartialEq)]
+pub struct LexicalError {
+    /// The type of error that occurred.
+    pub error: LexicalErrorType,
+    /// The location of the error.
+    pub location: Location,
+}
+
+impl LexicalError {
+    /// Creates a new `LexicalError` with the given error type and location.
+    pub fn new(error: LexicalErrorType, location: Location) -> Self {
+        Self { error, location }
+    }
+}
+
+/// Represents the different types of errors that can occur during lexing.
+#[derive(Debug, PartialEq)]
+pub enum LexicalErrorType {
+    // TODO: Can probably be removed, the places it is used seem to be able
+    // to use the `UnicodeError` variant instead.
+    #[doc(hidden)]
+    StringError,
+    // TODO: Should take a start/end position to report.
+    /// Decoding of a unicode escape sequence in a string literal failed.
+    UnicodeError,
+    /// The nesting of brackets/braces/parentheses is not balanced.
+    NestingError,
+    /// The indentation is not consistent.
+    IndentationError,
+    /// Inconsistent use of tabs and spaces.
+    TabError,
+    /// Encountered a tab after a space.
+    TabsAfterSpaces,
+    /// A non-default argument follows a default argument.
+    DefaultArgumentError,
+    /// A duplicate argument was found in a function definition.
+    DuplicateArgumentError(String),
+    /// A positional argument follows a keyword argument.
+    PositionalArgumentError,
+    /// An iterable argument unpacking `*args` follows keyword argument unpacking `**kwargs`.
+    UnpackedArgumentError,
+    /// A keyword argument was repeated.
+    DuplicateKeywordArgumentError(String),
+    /// An unrecognized token was encountered.
+    UnrecognizedToken { tok: char },
+    /// An f-string error containing the [`FStringErrorType`].
+    FStringError(FStringErrorType),
+    /// An unexpected character was encountered after a line continuation.
+    LineContinuationError,
+    /// An unexpected end of file was encountered.
+    Eof,
+    /// An unexpected error occurred.
+    OtherError(String),
+}
+
+impl std::fmt::Display for LexicalErrorType {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            LexicalErrorType::StringError => write!(f, "Got unexpected string"),
+            LexicalErrorType::FStringError(error) => write!(f, "f-string: {error}"),
+            LexicalErrorType::UnicodeError => write!(f, "Got unexpected unicode"),
+            LexicalErrorType::NestingError => write!(f, "Got unexpected nesting"),
+            LexicalErrorType::IndentationError => {
+                write!(f, "unindent does not match any outer indentation level")
+            }
+            LexicalErrorType::TabError => {
+                write!(f, "inconsistent use of tabs and spaces in indentation")
+            }
+            LexicalErrorType::TabsAfterSpaces => {
+                write!(f, "Tabs not allowed as part of indentation after spaces")
+            }
+            LexicalErrorType::DefaultArgumentError => {
+                write!(f, "non-default argument follows default argument")
+            }
+            LexicalErrorType::DuplicateArgumentError(arg_name) => {
+                write!(f, "duplicate argument '{arg_name}' in function definition")
+            }
+            LexicalErrorType::DuplicateKeywordArgumentError(arg_name) => {
+                write!(f, "keyword argument repeated: {arg_name}")
+            }
+            LexicalErrorType::PositionalArgumentError => {
+                write!(f, "positional argument follows keyword argument")
+            }
+            LexicalErrorType::UnpackedArgumentError => {
+                write!(
+                    f,
+                    "iterable argument unpacking follows keyword argument unpacking"
+                )
+            }
+            LexicalErrorType::UnrecognizedToken { tok } => {
+                write!(f, "Got unexpected token {tok}")
+            }
+            LexicalErrorType::LineContinuationError => {
+                write!(f, "unexpected character after line continuation character")
+            }
+            LexicalErrorType::Eof => write!(f, "unexpected EOF while parsing"),
+            LexicalErrorType::OtherError(msg) => write!(f, "{msg}"),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::{make_tokenizer, StringKind, Tok};
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index bfb56d74..440b6f67 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -124,7 +124,6 @@
 extern crate log;
 pub use rustpython_ast as ast;
 
-pub mod error;
 mod function;
 pub mod lexer;
 pub mod mode;
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index e0865b2d..1854c25a 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -12,13 +12,15 @@
 //! [Abstract Syntax Tree]: https://en.wikipedia.org/wiki/Abstract_syntax_tree
 //! [`Mode`]: crate::mode
 
-use crate::lexer::{LexResult, Tok};
+use crate::lexer::{LexResult, LexicalError, LexicalErrorType, Tok};
 pub use crate::mode::Mode;
-use crate::{ast, error::ParseError, lexer, python};
+use crate::{ast, lexer, python};
 use ast::Location;
 use itertools::Itertools;
 use std::iter;
 
+pub(super) use lalrpop_util::ParseError as LalrpopError;
+
 /// Parse a full Python program usually consisting of multiple lines.
 ///  
 /// This is a convenience function that can be used to parse a full Python program without having to
@@ -194,7 +196,124 @@ pub fn parse_tokens(
         .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
     python::TopParser::new()
         .parse(tokenizer.into_iter())
-        .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
+        .map_err(|e| parse_error_from_lalrpop(e, source_path))
+}
+
+/// Represents represent errors that occur during parsing and are
+/// returned by the `parse_*` functions in the [parser] module.
+///
+/// [parser]: crate::parser
+pub type ParseError = rustpython_compiler_core::BaseError<ParseErrorType>;
+
+/// Represents the different types of errors that can occur during parsing.
+#[derive(Debug, PartialEq, thiserror::Error)]
+pub enum ParseErrorType {
+    /// Parser encountered an unexpected end of input
+    Eof,
+    /// Parser encountered an extra token
+    ExtraToken(Tok),
+    /// Parser encountered an invalid token
+    InvalidToken,
+    /// Parser encountered an unexpected token
+    UnrecognizedToken(Tok, Option<String>),
+    // Maps to `User` type from `lalrpop-util`
+    /// Parser encountered an error during lexing.
+    Lexical(LexicalErrorType),
+}
+
+// Convert `lalrpop_util::ParseError` to our internal type
+fn parse_error_from_lalrpop(
+    err: LalrpopError<Location, Tok, LexicalError>,
+    source_path: &str,
+) -> ParseError {
+    let source_path = source_path.to_owned();
+    match err {
+        // TODO: Are there cases where this isn't an EOF?
+        LalrpopError::InvalidToken { location } => ParseError {
+            error: ParseErrorType::Eof,
+            location,
+            source_path,
+        },
+        LalrpopError::ExtraToken { token } => ParseError {
+            error: ParseErrorType::ExtraToken(token.1),
+            location: token.0,
+            source_path,
+        },
+        LalrpopError::User { error } => ParseError {
+            error: ParseErrorType::Lexical(error.error),
+            location: error.location,
+            source_path,
+        },
+        LalrpopError::UnrecognizedToken { token, expected } => {
+            // Hacky, but it's how CPython does it. See PyParser_AddToken,
+            // in particular "Only one possible expected token" comment.
+            let expected = (expected.len() == 1).then(|| expected[0].clone());
+            ParseError {
+                error: ParseErrorType::UnrecognizedToken(token.1, expected),
+                location: token.0.with_col_offset(1),
+                source_path,
+            }
+        }
+        LalrpopError::UnrecognizedEOF { location, expected } => {
+            // This could be an initial indentation error that we should ignore
+            let indent_error = expected == ["Indent"];
+            if indent_error {
+                ParseError {
+                    error: ParseErrorType::Lexical(LexicalErrorType::IndentationError),
+                    location,
+                    source_path,
+                }
+            } else {
+                ParseError {
+                    error: ParseErrorType::Eof,
+                    location,
+                    source_path,
+                }
+            }
+        }
+    }
+}
+
+impl std::fmt::Display for ParseErrorType {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match *self {
+            ParseErrorType::Eof => write!(f, "Got unexpected EOF"),
+            ParseErrorType::ExtraToken(ref tok) => write!(f, "Got extraneous token: {tok:?}"),
+            ParseErrorType::InvalidToken => write!(f, "Got invalid token"),
+            ParseErrorType::UnrecognizedToken(ref tok, ref expected) => {
+                if *tok == Tok::Indent {
+                    write!(f, "unexpected indent")
+                } else if expected.as_deref() == Some("Indent") {
+                    write!(f, "expected an indented block")
+                } else {
+                    write!(f, "invalid syntax. Got unexpected token {tok}")
+                }
+            }
+            ParseErrorType::Lexical(ref error) => write!(f, "{error}"),
+        }
+    }
+}
+
+impl ParseErrorType {
+    /// Returns true if the error is an indentation error.
+    pub fn is_indentation_error(&self) -> bool {
+        match self {
+            ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true,
+            ParseErrorType::UnrecognizedToken(token, expected) => {
+                *token == Tok::Indent || expected.clone() == Some("Indent".to_owned())
+            }
+            _ => false,
+        }
+    }
+
+    /// Returns true if the error is a tab error.
+    pub fn is_tab_error(&self) -> bool {
+        matches!(
+            self,
+            ParseErrorType::Lexical(LexicalErrorType::TabError)
+                | ParseErrorType::Lexical(LexicalErrorType::TabsAfterSpaces)
+        )
+    }
 }
 
 #[cfg(test)]
diff --git a/parser/src/string.rs b/parser/src/string.rs
index 84c6fe22..01431777 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -3,15 +3,14 @@
 // The lexer doesn't do any special handling of f-strings, it just treats them as
 // regular strings. Since the parser has no definition of f-string formats (Pending PEP 701)
 // we have to do the parsing here, manually.
-use itertools::Itertools;
-
 use self::FStringErrorType::*;
 use crate::{
     ast::{Constant, ConversionFlag, Expr, ExprKind, Location},
-    error::{FStringError, FStringErrorType, LexicalError, LexicalErrorType, ParseError},
-    parser::parse_expression_located,
-    token::StringKind,
+    lexer::{LexicalError, LexicalErrorType},
+    parser::{parse_expression_located, LalrpopError, ParseError, ParseErrorType},
+    token::{StringKind, Tok},
 };
+use itertools::Itertools;
 use std::{iter, str};
 
 // unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
@@ -651,6 +650,107 @@ pub(crate) fn parse_strings(
     ))
 }
 
+// TODO: consolidate these with ParseError
+/// An error that occurred during parsing of an f-string.
+#[derive(Debug, PartialEq)]
+pub struct FStringError {
+    /// The type of error that occurred.
+    pub error: FStringErrorType,
+    /// The location of the error.
+    pub location: Location,
+}
+
+impl FStringError {
+    /// Creates a new `FStringError` with the given error type and location.
+    pub fn new(error: FStringErrorType, location: Location) -> Self {
+        Self { error, location }
+    }
+}
+
+impl From<FStringError> for LexicalError {
+    fn from(err: FStringError) -> Self {
+        LexicalError {
+            error: LexicalErrorType::FStringError(err.error),
+            location: err.location,
+        }
+    }
+}
+
+/// Represents the different types of errors that can occur during parsing of an f-string.
+#[derive(Debug, PartialEq)]
+pub enum FStringErrorType {
+    /// Expected a right brace after an opened left brace.
+    UnclosedLbrace,
+    /// Expected a left brace after an ending right brace.
+    UnopenedRbrace,
+    /// Expected a right brace after a conversion flag.
+    ExpectedRbrace,
+    /// An error occurred while parsing an f-string expression.
+    InvalidExpression(Box<ParseErrorType>),
+    /// An invalid conversion flag was encountered.
+    InvalidConversionFlag,
+    /// An empty expression was encountered.
+    EmptyExpression,
+    /// An opening delimiter was not closed properly.
+    MismatchedDelimiter(char, char),
+    /// Too many nested expressions in an f-string.
+    ExpressionNestedTooDeeply,
+    /// The f-string expression cannot include the given character.
+    ExpressionCannotInclude(char),
+    /// A single right brace was encountered.
+    SingleRbrace,
+    /// A closing delimiter was not opened properly.
+    Unmatched(char),
+    // TODO: Test this case.
+    /// Unterminated string.
+    UnterminatedString,
+}
+
+impl std::fmt::Display for FStringErrorType {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            FStringErrorType::UnclosedLbrace => write!(f, "expecting '}}'"),
+            FStringErrorType::UnopenedRbrace => write!(f, "Unopened '}}'"),
+            FStringErrorType::ExpectedRbrace => write!(f, "Expected '}}' after conversion flag."),
+            FStringErrorType::InvalidExpression(error) => {
+                write!(f, "{error}")
+            }
+            FStringErrorType::InvalidConversionFlag => write!(f, "invalid conversion character"),
+            FStringErrorType::EmptyExpression => write!(f, "empty expression not allowed"),
+            FStringErrorType::MismatchedDelimiter(first, second) => write!(
+                f,
+                "closing parenthesis '{second}' does not match opening parenthesis '{first}'"
+            ),
+            FStringErrorType::SingleRbrace => write!(f, "single '}}' is not allowed"),
+            FStringErrorType::Unmatched(delim) => write!(f, "unmatched '{delim}'"),
+            FStringErrorType::ExpressionNestedTooDeeply => {
+                write!(f, "expressions nested too deeply")
+            }
+            FStringErrorType::UnterminatedString => {
+                write!(f, "unterminated string")
+            }
+            FStringErrorType::ExpressionCannotInclude(c) => {
+                if *c == '\\' {
+                    write!(f, "f-string expression part cannot include a backslash")
+                } else {
+                    write!(f, "f-string expression part cannot include '{c}'s")
+                }
+            }
+        }
+    }
+}
+
+impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
+    fn from(err: FStringError) -> Self {
+        lalrpop_util::ParseError::User {
+            error: LexicalError {
+                error: LexicalErrorType::FStringError(err.error),
+                location: err.location,
+            },
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/src/lib.rs b/src/lib.rs
index 657d0fe7..275815d9 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,8 +1,8 @@
 use rustpython_codegen::{compile, symboltable};
 use rustpython_parser::{
     ast::{fold::Fold, ConstantOptimizer},
-    error::ParseErrorType,
     parser,
+    parser::ParseErrorType,
 };
 
 pub use rustpython_codegen::compile::CompileOpts;
@@ -13,12 +13,12 @@ pub enum CompileErrorType {
     #[error(transparent)]
     Codegen(#[from] rustpython_codegen::error::CodegenErrorType),
     #[error(transparent)]
-    Parse(#[from] rustpython_parser::error::ParseErrorType),
+    Parse(#[from] rustpython_parser::parser::ParseErrorType),
 }
 
 pub type CompileError = rustpython_compiler_core::CompileError<CompileErrorType>;
 
-fn error_from_parse(error: rustpython_parser::error::ParseError, source: &str) -> CompileError {
+fn error_from_parse(error: rustpython_parser::parser::ParseError, source: &str) -> CompileError {
     let error: CompileErrorBody<ParseErrorType> = error.into();
     CompileError::from(error, source)
 }

From 019faeac2428152acbcacb0db9f61f64233c0f9e Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 22 Feb 2023 16:52:40 +0900
Subject: [PATCH 428/459] Fix ModeParseError message

---
 core/src/mode.rs   | 8 +++-----
 parser/src/mode.rs | 8 +++-----
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/core/src/mode.rs b/core/src/mode.rs
index b56f226a..ca0d5992 100644
--- a/core/src/mode.rs
+++ b/core/src/mode.rs
@@ -15,18 +15,16 @@ impl std::str::FromStr for Mode {
             "exec" => Ok(Mode::Exec),
             "eval" => Ok(Mode::Eval),
             "single" => Ok(Mode::Single),
-            _ => Err(ModeParseError { _priv: () }),
+            _ => Err(ModeParseError(())),
         }
     }
 }
 
 #[derive(Debug)]
-pub struct ModeParseError {
-    _priv: (),
-}
+pub struct ModeParseError(());
 
 impl std::fmt::Display for ModeParseError {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, r#"mode should be "exec", "eval", or "single""#)
+        write!(f, r#"mode must be "exec", "eval", or "single""#)
     }
 }
diff --git a/parser/src/mode.rs b/parser/src/mode.rs
index 4403fbee..9e9f1e1f 100644
--- a/parser/src/mode.rs
+++ b/parser/src/mode.rs
@@ -39,19 +39,17 @@ impl std::str::FromStr for Mode {
         match s {
             "exec" | "single" => Ok(Mode::Module),
             "eval" => Ok(Mode::Expression),
-            _ => Err(ModeParseError { _priv: () }),
+            _ => Err(ModeParseError(())),
         }
     }
 }
 
 /// Returned when a given mode is not valid.
 #[derive(Debug)]
-pub struct ModeParseError {
-    _priv: (),
-}
+pub struct ModeParseError(());
 
 impl std::fmt::Display for ModeParseError {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-        write!(f, r#"mode should be "exec", "eval", or "single""#)
+        write!(f, r#"mode must be "exec", "eval", or "single""#)
     }
 }

From b5829b547ace1fcd696f7137c4ec1a0841829188 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 22 Feb 2023 17:28:49 +0900
Subject: [PATCH 429/459] relocate feature-independent use

---
 ast/src/constant.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ast/src/constant.rs b/ast/src/constant.rs
index 54aa7594..55ee09a8 100644
--- a/ast/src/constant.rs
+++ b/ast/src/constant.rs
@@ -1,5 +1,3 @@
-use std::fmt::Error;
-
 use num_bigint::BigInt;
 pub use rustpython_compiler_core::ConversionFlag;
 
@@ -44,7 +42,9 @@ impl std::fmt::Display for Constant {
             Constant::None => f.pad("None"),
             Constant::Bool(b) => f.pad(if *b { "True" } else { "False" }),
             Constant::Str(s) => rustpython_common::str::repr(s).fmt(f),
-            Constant::Bytes(b) => f.pad(&rustpython_common::bytes::repr(b).map_err(|_err| Error)?),
+            Constant::Bytes(b) => {
+                f.pad(&rustpython_common::bytes::repr(b).map_err(|_err| std::fmt::Error)?)
+            }
             Constant::Int(i) => i.fmt(f),
             Constant::Tuple(tup) => {
                 if let [elt] = &**tup {

From a8d4de27ede1dcdfcd016f2dbd6aa3b38a6782c5 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 22 Feb 2023 15:58:51 +0900
Subject: [PATCH 430/459] make_tokenizer -> lex to integrate terms

we don't distinguish scanner or tokenizer from lexer
---
 parser/src/lexer.rs         | 30 +++++++++++++++---------------
 parser/src/lib.rs           |  8 ++++----
 parser/src/parser.rs        | 12 ++++++------
 parser/src/soft_keywords.rs |  4 ++--
 4 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 7da3b8c5..fb0a9092 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -4,7 +4,7 @@
 //! governing what is and is not a valid token are defined in the Python reference
 //! guide section on [Lexical analysis].
 //!
-//! The primary function in this module is [`make_tokenizer`], which takes a string slice
+//! The primary function in this module is [`lex`], which takes a string slice
 //! and returns an iterator over the tokens in the source code. The tokens are currently returned
 //! as a `Result<Spanned, LexicalError>`, where [`Spanned`] is a tuple containing the
 //! start and end [`Location`] and a [`Tok`] denoting the token.
@@ -12,12 +12,12 @@
 //! # Example
 //!
 //! ```
-//! use rustpython_parser::lexer::{make_tokenizer, Tok};
+//! use rustpython_parser::lexer::{lex, Tok};
 //! use rustpython_parser::mode::Mode;
 //! use rustpython_parser::token::StringKind;
 //!
 //! let source = "x = 'RustPython'";
-//! let tokens = make_tokenizer(source, Mode::Module)
+//! let tokens = lex(source, Mode::Module)
 //!     .map(|tok| tok.expect("Failed to lex"))
 //!     .collect::<Vec<_>>();
 //!
@@ -195,29 +195,29 @@ pub type Spanned = (Location, Tok, Location);
 /// The result of lexing a token.
 pub type LexResult = Result<Spanned, LexicalError>;
 
-/// Create a new tokenizer from a source string.
+/// Create a new lexer from a source string.
 ///
 /// # Examples
 ///
 /// ```
 /// use rustpython_parser::mode::Mode;
-/// use rustpython_parser::lexer::{make_tokenizer};
+/// use rustpython_parser::lexer::{lex};
 ///
 /// let source = "def hello(): return 'world'";
-/// let tokenizer = make_tokenizer(source, Mode::Module);
+/// let lexer = lex(source, Mode::Module);
 ///
-/// for token in tokenizer {
+/// for token in lexer {
 ///    println!("{:?}", token);
 /// }
 /// ```
 #[inline]
-pub fn make_tokenizer(source: &str, mode: Mode) -> impl Iterator<Item = LexResult> + '_ {
-    make_tokenizer_located(source, mode, Location::default())
+pub fn lex(source: &str, mode: Mode) -> impl Iterator<Item = LexResult> + '_ {
+    lex_located(source, mode, Location::default())
 }
 
-/// Create a new tokenizer from a source string, starting at a given location.
-/// You probably want to use [`make_tokenizer`] instead.
-pub fn make_tokenizer_located(
+/// Create a new lexer from a source string, starting at a given location.
+/// You probably want to use [`lex`] instead.
+pub fn lex_located(
     source: &str,
     mode: Mode,
     start_location: Location,
@@ -230,7 +230,7 @@ where
     T: Iterator<Item = char>,
 {
     /// Create a new lexer from T and a starting location. You probably want to use
-    /// [`make_tokenizer`] instead.
+    /// [`lex`] instead.
     pub fn new(input: T, start: Location) -> Self {
         let mut lxr = Lexer {
             at_begin_of_line: true,
@@ -1320,7 +1320,7 @@ impl std::fmt::Display for LexicalErrorType {
 
 #[cfg(test)]
 mod tests {
-    use super::{make_tokenizer, StringKind, Tok};
+    use super::{lex, StringKind, Tok};
     use crate::mode::Mode;
     use num_bigint::BigInt;
 
@@ -1329,7 +1329,7 @@ mod tests {
     const UNIX_EOL: &str = "\n";
 
     pub fn lex_source(source: &str) -> Vec<Tok> {
-        let lexer = make_tokenizer(source, Mode::Module);
+        let lexer = lex(source, Mode::Module);
         lexer.map(|x| x.unwrap().1).collect()
     }
 
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index 440b6f67..67692461 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -67,20 +67,20 @@
 //!
 //! ```
 //! use rustpython_parser::mode::Mode;
-//! use rustpython_parser::lexer::make_tokenizer;
+//! use rustpython_parser::lexer::lex;
 //!
 //! let python_source = r#"
 //! def is_odd(i):
 //!     return bool(i & 1)
 //! "#;
-//! let mut tokens = make_tokenizer(python_source, Mode::Module);
+//! let mut tokens = lex(python_source, Mode::Module);
 //! assert!(tokens.all(|t| t.is_ok()));
 //! ```
 //!
 //! These tokens can be directly fed into the parser to generate an AST:
 //!
 //! ```
-//! use rustpython_parser::lexer::make_tokenizer;
+//! use rustpython_parser::lexer::lex;
 //! use rustpython_parser::mode::Mode;
 //! use rustpython_parser::parser::parse_tokens;
 //!
@@ -88,7 +88,7 @@
 //! def is_odd(i):
 //!    return bool(i & 1)
 //! "#;
-//! let tokens = make_tokenizer(python_source, Mode::Module);
+//! let tokens = lex(python_source, Mode::Module);
 //! let ast = parse_tokens(tokens, Mode::Module, "<embedded>");
 //!
 //! assert!(ast.is_ok());
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 1854c25a..2cdb7c20 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -164,7 +164,7 @@ pub fn parse_located(
     source_path: &str,
     location: Location,
 ) -> Result<ast::Mod, ParseError> {
-    let lxr = lexer::make_tokenizer_located(source, mode, location);
+    let lxr = lexer::lex_located(source, mode, location);
     parse_tokens(lxr, mode, source_path)
 }
 
@@ -175,14 +175,14 @@ pub fn parse_located(
 /// # Example
 ///
 /// As an example, instead of parsing a string, we can parse a list of tokens after we generate
-/// them using the [`lexer::make_tokenizer`] function:
+/// them using the [`lexer::lex`] function:
 ///
 /// ```
-/// use rustpython_parser::lexer::make_tokenizer;
+/// use rustpython_parser::lexer::lex;
 /// use rustpython_parser::mode::Mode;
 /// use rustpython_parser::parser::parse_tokens;
 ///
-/// let expr = parse_tokens(make_tokenizer("1 + 2", Mode::Expression), Mode::Expression, "<embedded>");
+/// let expr = parse_tokens(lex("1 + 2", Mode::Expression), Mode::Expression, "<embedded>");
 /// assert!(expr.is_ok());
 /// ```
 pub fn parse_tokens(
@@ -191,11 +191,11 @@ pub fn parse_tokens(
     source_path: &str,
 ) -> Result<ast::Mod, ParseError> {
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
-    let tokenizer = iter::once(Ok(marker_token))
+    let lexer = iter::once(Ok(marker_token))
         .chain(lxr)
         .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
     python::TopParser::new()
-        .parse(tokenizer.into_iter())
+        .parse(lexer.into_iter())
         .map_err(|e| parse_error_from_lalrpop(e, source_path))
 }
 
diff --git a/parser/src/soft_keywords.rs b/parser/src/soft_keywords.rs
index a029ccd4..7011f3a7 100644
--- a/parser/src/soft_keywords.rs
+++ b/parser/src/soft_keywords.rs
@@ -27,9 +27,9 @@ impl<I> SoftKeywordTransformer<I>
 where
     I: Iterator<Item = LexResult>,
 {
-    pub fn new(tokenizer: I, mode: Mode) -> Self {
+    pub fn new(lexer: I, mode: Mode) -> Self {
         Self {
-            underlying: tokenizer.multipeek(),
+            underlying: lexer.multipeek(),
             start_of_line: matches!(mode, Mode::Interactive | Mode::Module),
         }
     }

From bf3f3eca62cfe86638833b42a97d4bebfaabe855 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 22 Feb 2023 17:14:20 +0900
Subject: [PATCH 431/459] Flatten rustpython_parser interface

---
 codegen/src/compile.rs      |   4 +-
 parser/python.lalrpop       | 197 ++++++++++++++++++------------------
 parser/src/function.rs      |   6 +-
 parser/src/lexer.rs         |  31 +++---
 parser/src/lib.rs           |  35 +++----
 parser/src/parser.rs        |  36 +++----
 parser/src/soft_keywords.rs |   4 +-
 parser/src/string.rs        |  36 ++++---
 src/lib.rs                  |   9 +-
 9 files changed, 174 insertions(+), 184 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index e3c17bcd..92ed523c 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -86,7 +86,7 @@ impl CompileContext {
     }
 }
 
-/// Compile an ast::Mod produced from rustpython_parser::parser::parse()
+/// Compile an ast::Mod produced from rustpython_parser::parse()
 pub fn compile_top(
     ast: &ast::Mod,
     source_path: String,
@@ -2846,7 +2846,7 @@ mod tests {
     use super::{CompileOpts, Compiler};
     use crate::symboltable::SymbolTable;
     use rustpython_compiler_core::CodeObject;
-    use rustpython_parser::parser;
+    use rustpython_parser as parser;
 
     fn compile_exec(source: &str) -> CodeObject {
         let mut compiler: Compiler = Compiler::new(
diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 57d5dbe0..3ca06b5b 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -7,10 +7,9 @@ use crate::{
     ast,
     lexer::{LexicalError, LexicalErrorType},
     function::{ArgumentList, parse_args, parse_params, validate_arguments},
-    lexer,
     context::set_context,
     string::parse_strings,
-    token::StringKind,
+    token::{self, StringKind},
 };
 use num_bigint::BigInt;
 
@@ -1937,106 +1936,106 @@ extern {
     type Location = ast::Location;
     type Error = LexicalError;
 
-    enum lexer::Tok {
-        Indent => lexer::Tok::Indent,
-        Dedent => lexer::Tok::Dedent,
-        StartModule => lexer::Tok::StartModule,
-        StartInteractive => lexer::Tok::StartInteractive,
-        StartExpression => lexer::Tok::StartExpression,
-        "+" => lexer::Tok::Plus,
-        "-" => lexer::Tok::Minus,
-        "~" => lexer::Tok::Tilde,
-        ":" => lexer::Tok::Colon,
-        "." => lexer::Tok::Dot,
-        "..." => lexer::Tok::Ellipsis,
-        "," => lexer::Tok::Comma,
-        "*" => lexer::Tok::Star,
-        "**" => lexer::Tok::DoubleStar,
-        "&" => lexer::Tok::Amper,
-        "@" => lexer::Tok::At,
-        "%" => lexer::Tok::Percent,
-        "//" => lexer::Tok::DoubleSlash,
-        "^" => lexer::Tok::CircumFlex,
-        "|" => lexer::Tok::Vbar,
-        "<<" => lexer::Tok::LeftShift,
-        ">>" => lexer::Tok::RightShift,
-        "/" => lexer::Tok::Slash,
-        "(" => lexer::Tok::Lpar,
-        ")" => lexer::Tok::Rpar,
-        "[" => lexer::Tok::Lsqb,
-        "]" => lexer::Tok::Rsqb,
-        "{" => lexer::Tok::Lbrace,
-        "}" => lexer::Tok::Rbrace,
-        "=" => lexer::Tok::Equal,
-        "+=" => lexer::Tok::PlusEqual,
-        "-=" => lexer::Tok::MinusEqual,
-        "*=" => lexer::Tok::StarEqual,
-        "@=" => lexer::Tok::AtEqual,
-        "/=" => lexer::Tok::SlashEqual,
-        "%=" => lexer::Tok::PercentEqual,
-        "&=" => lexer::Tok::AmperEqual,
-        "|=" => lexer::Tok::VbarEqual,
-        "^=" => lexer::Tok::CircumflexEqual,
-        "<<=" => lexer::Tok::LeftShiftEqual,
-        ">>=" => lexer::Tok::RightShiftEqual,
-        "**=" => lexer::Tok::DoubleStarEqual,
-        "//=" => lexer::Tok::DoubleSlashEqual,
-        ":=" => lexer::Tok::ColonEqual,
-        "==" => lexer::Tok::EqEqual,
-        "!=" => lexer::Tok::NotEqual,
-        "<" => lexer::Tok::Less,
-        "<=" => lexer::Tok::LessEqual,
-        ">" => lexer::Tok::Greater,
-        ">=" => lexer::Tok::GreaterEqual,
-        "->" => lexer::Tok::Rarrow,
-        "and" => lexer::Tok::And,
-        "as" => lexer::Tok::As,
-        "assert" => lexer::Tok::Assert,
-        "async" => lexer::Tok::Async,
-        "await" => lexer::Tok::Await,
-        "break" => lexer::Tok::Break,
-        "class" => lexer::Tok::Class,
-        "continue" => lexer::Tok::Continue,
-        "def" => lexer::Tok::Def,
-        "del" => lexer::Tok::Del,
-        "elif" => lexer::Tok::Elif,
-        "else" => lexer::Tok::Else,
-        "except" => lexer::Tok::Except,
-        "finally" => lexer::Tok::Finally,
-        "for" => lexer::Tok::For,
-        "from" => lexer::Tok::From,
-        "global" => lexer::Tok::Global,
-        "if" => lexer::Tok::If,
-        "import" => lexer::Tok::Import,
-        "in" => lexer::Tok::In,
-        "is" => lexer::Tok::Is,
-        "lambda" => lexer::Tok::Lambda,
-        "nonlocal" => lexer::Tok::Nonlocal,
-        "not" => lexer::Tok::Not,
-        "or" => lexer::Tok::Or,
-        "pass" => lexer::Tok::Pass,
-        "raise" => lexer::Tok::Raise,
-        "return" => lexer::Tok::Return,
-        "try" => lexer::Tok::Try,
-        "while" => lexer::Tok::While,
-        "match" => lexer::Tok::Match,
-        "case" => lexer::Tok::Case,
-        "with" => lexer::Tok::With,
-        "yield" => lexer::Tok::Yield,
-        "True" => lexer::Tok::True,
-        "False" => lexer::Tok::False,
-        "None" => lexer::Tok::None,
-        int => lexer::Tok::Int { value: <BigInt> },
-        float => lexer::Tok::Float { value: <f64> },
-        complex => lexer::Tok::Complex { real: <f64>, imag: <f64> },
-        string => lexer::Tok::String {
+    enum token::Tok {
+        Indent => token::Tok::Indent,
+        Dedent => token::Tok::Dedent,
+        StartModule => token::Tok::StartModule,
+        StartInteractive => token::Tok::StartInteractive,
+        StartExpression => token::Tok::StartExpression,
+        "+" => token::Tok::Plus,
+        "-" => token::Tok::Minus,
+        "~" => token::Tok::Tilde,
+        ":" => token::Tok::Colon,
+        "." => token::Tok::Dot,
+        "..." => token::Tok::Ellipsis,
+        "," => token::Tok::Comma,
+        "*" => token::Tok::Star,
+        "**" => token::Tok::DoubleStar,
+        "&" => token::Tok::Amper,
+        "@" => token::Tok::At,
+        "%" => token::Tok::Percent,
+        "//" => token::Tok::DoubleSlash,
+        "^" => token::Tok::CircumFlex,
+        "|" => token::Tok::Vbar,
+        "<<" => token::Tok::LeftShift,
+        ">>" => token::Tok::RightShift,
+        "/" => token::Tok::Slash,
+        "(" => token::Tok::Lpar,
+        ")" => token::Tok::Rpar,
+        "[" => token::Tok::Lsqb,
+        "]" => token::Tok::Rsqb,
+        "{" => token::Tok::Lbrace,
+        "}" => token::Tok::Rbrace,
+        "=" => token::Tok::Equal,
+        "+=" => token::Tok::PlusEqual,
+        "-=" => token::Tok::MinusEqual,
+        "*=" => token::Tok::StarEqual,
+        "@=" => token::Tok::AtEqual,
+        "/=" => token::Tok::SlashEqual,
+        "%=" => token::Tok::PercentEqual,
+        "&=" => token::Tok::AmperEqual,
+        "|=" => token::Tok::VbarEqual,
+        "^=" => token::Tok::CircumflexEqual,
+        "<<=" => token::Tok::LeftShiftEqual,
+        ">>=" => token::Tok::RightShiftEqual,
+        "**=" => token::Tok::DoubleStarEqual,
+        "//=" => token::Tok::DoubleSlashEqual,
+        ":=" => token::Tok::ColonEqual,
+        "==" => token::Tok::EqEqual,
+        "!=" => token::Tok::NotEqual,
+        "<" => token::Tok::Less,
+        "<=" => token::Tok::LessEqual,
+        ">" => token::Tok::Greater,
+        ">=" => token::Tok::GreaterEqual,
+        "->" => token::Tok::Rarrow,
+        "and" => token::Tok::And,
+        "as" => token::Tok::As,
+        "assert" => token::Tok::Assert,
+        "async" => token::Tok::Async,
+        "await" => token::Tok::Await,
+        "break" => token::Tok::Break,
+        "class" => token::Tok::Class,
+        "continue" => token::Tok::Continue,
+        "def" => token::Tok::Def,
+        "del" => token::Tok::Del,
+        "elif" => token::Tok::Elif,
+        "else" => token::Tok::Else,
+        "except" => token::Tok::Except,
+        "finally" => token::Tok::Finally,
+        "for" => token::Tok::For,
+        "from" => token::Tok::From,
+        "global" => token::Tok::Global,
+        "if" => token::Tok::If,
+        "import" => token::Tok::Import,
+        "in" => token::Tok::In,
+        "is" => token::Tok::Is,
+        "lambda" => token::Tok::Lambda,
+        "nonlocal" => token::Tok::Nonlocal,
+        "not" => token::Tok::Not,
+        "or" => token::Tok::Or,
+        "pass" => token::Tok::Pass,
+        "raise" => token::Tok::Raise,
+        "return" => token::Tok::Return,
+        "try" => token::Tok::Try,
+        "while" => token::Tok::While,
+        "match" => token::Tok::Match,
+        "case" => token::Tok::Case,
+        "with" => token::Tok::With,
+        "yield" => token::Tok::Yield,
+        "True" => token::Tok::True,
+        "False" => token::Tok::False,
+        "None" => token::Tok::None,
+        int => token::Tok::Int { value: <BigInt> },
+        float => token::Tok::Float { value: <f64> },
+        complex => token::Tok::Complex { real: <f64>, imag: <f64> },
+        string => token::Tok::String {
             value: <String>,
             kind: <StringKind>,
             triple_quoted: <bool>
         },
-        name => lexer::Tok::Name { name: <String> },
-        "\n" => lexer::Tok::Newline,
-        ";" => lexer::Tok::Semi,
-        "#" => lexer::Tok::Comment(_),
+        name => token::Tok::Name { name: <String> },
+        "\n" => token::Tok::Newline,
+        ";" => token::Tok::Semi,
+        "#" => token::Tok::Comment(_),
     }
 }
diff --git a/parser/src/function.rs b/parser/src/function.rs
index dc950396..cfda1c51 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -1,7 +1,9 @@
 // Contains functions that perform validation and parsing of arguments and parameters.
 // Checks apply both to functions and to lambdas.
-use crate::ast;
-use crate::lexer::{LexicalError, LexicalErrorType};
+use crate::{
+    ast,
+    lexer::{LexicalError, LexicalErrorType},
+};
 use rustc_hash::FxHashSet;
 
 pub(crate) struct ArgumentList {
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index fb0a9092..f4862b3a 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -12,9 +12,7 @@
 //! # Example
 //!
 //! ```
-//! use rustpython_parser::lexer::{lex, Tok};
-//! use rustpython_parser::mode::Mode;
-//! use rustpython_parser::token::StringKind;
+//! use rustpython_parser::{lexer::lex, Tok, Mode, StringKind};
 //!
 //! let source = "x = 'RustPython'";
 //! let tokens = lex(source, Mode::Module)
@@ -33,19 +31,16 @@
 //! ```
 //!
 //! [Lexical analysis]: https://docs.python.org/3/reference/lexical_analysis.html
-pub use super::token::{StringKind, Tok};
-use crate::ast::Location;
-use crate::mode::Mode;
-use crate::soft_keywords::SoftKeywordTransformer;
-use crate::string::FStringErrorType;
+use crate::{
+    ast::Location,
+    mode::Mode,
+    soft_keywords::SoftKeywordTransformer,
+    string::FStringErrorType,
+    token::{StringKind, Tok},
+};
 use num_bigint::BigInt;
-use num_traits::identities::Zero;
-use num_traits::Num;
-use std::char;
-use std::cmp::Ordering;
-use std::ops::Index;
-use std::slice::SliceIndex;
-use std::str::FromStr;
+use num_traits::{Num, Zero};
+use std::{char, cmp::Ordering, ops::Index, slice::SliceIndex, str::FromStr};
 use unic_emoji_char::is_emoji_presentation;
 use unic_ucd_ident::{is_xid_continue, is_xid_start};
 
@@ -200,8 +195,7 @@ pub type LexResult = Result<Spanned, LexicalError>;
 /// # Examples
 ///
 /// ```
-/// use rustpython_parser::mode::Mode;
-/// use rustpython_parser::lexer::{lex};
+/// use rustpython_parser::{Mode, lexer::lex};
 ///
 /// let source = "def hello(): return 'world'";
 /// let lexer = lex(source, Mode::Module);
@@ -1320,8 +1314,7 @@ impl std::fmt::Display for LexicalErrorType {
 
 #[cfg(test)]
 mod tests {
-    use super::{lex, StringKind, Tok};
-    use crate::mode::Mode;
+    use super::*;
     use num_bigint::BigInt;
 
     const WINDOWS_EOL: &str = "\r\n";
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index 67692461..bc970740 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -54,20 +54,18 @@
 //!
 //! The functionality of this crate is split into several modules:
 //!
-//! - [token]: This module contains the definition of the tokens that are generated by the lexer.
+//! - token: This module contains the definition of the tokens that are generated by the lexer.
 //! - [lexer]: This module contains the lexer and is responsible for generating the tokens.
-//! - [parser]: This module contains an interface to the parser and is responsible for generating the AST.
+//! - parser: This module contains an interface to the parser and is responsible for generating the AST.
 //!     - Functions and strings have special parsing requirements that are handled in additional files.
-//! - [mode]: This module contains the definition of the different modes that the parser can be in.
-//! - [error]: This module contains the definition of the errors that can be returned by the parser.
+//! - mode: This module contains the definition of the different modes that the parser can be in.
 //!
 //! # Examples
 //!
 //! For example, to get a stream of tokens from a given string, one could do this:
 //!
 //! ```
-//! use rustpython_parser::mode::Mode;
-//! use rustpython_parser::lexer::lex;
+//! use rustpython_parser::{lexer::lex, Mode};
 //!
 //! let python_source = r#"
 //! def is_odd(i):
@@ -80,9 +78,7 @@
 //! These tokens can be directly fed into the parser to generate an AST:
 //!
 //! ```
-//! use rustpython_parser::lexer::lex;
-//! use rustpython_parser::mode::Mode;
-//! use rustpython_parser::parser::parse_tokens;
+//! use rustpython_parser::{lexer::lex, Mode, parse_tokens};
 //!
 //! let python_source = r#"
 //! def is_odd(i):
@@ -98,7 +94,7 @@
 //! mode or tokenizing the source beforehand:
 //!
 //! ```
-//! use rustpython_parser::parser::parse_program;
+//! use rustpython_parser::parse_program;
 //!
 //! let python_source = r#"
 //! def is_odd(i):
@@ -111,11 +107,7 @@
 //!
 //! [lexical analysis]: https://en.wikipedia.org/wiki/Lexical_analysis
 //! [parsing]: https://en.wikipedia.org/wiki/Parsing
-//! [token]: crate::token
 //! [lexer]: crate::lexer
-//! [parser]: crate::parser
-//! [mode]: crate::mode
-//! [error]: crate::error
 
 #![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
 #![doc(html_root_url = "https://docs.rs/rustpython-parser/")]
@@ -125,12 +117,21 @@ extern crate log;
 pub use rustpython_ast as ast;
 
 mod function;
+// Skip flattening lexer to distinguish from full parser
 pub mod lexer;
-pub mod mode;
-pub mod parser;
+mod mode;
+mod parser;
 mod string;
 #[rustfmt::skip]
 mod python;
 mod context;
 mod soft_keywords;
-pub mod token;
+mod token;
+
+pub use mode::Mode;
+pub use parser::{
+    parse, parse_expression, parse_expression_located, parse_located, parse_program, parse_tokens,
+    ParseError, ParseErrorType,
+};
+pub use string::FStringErrorType;
+pub use token::{StringKind, Tok};
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 2cdb7c20..12418486 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -12,10 +12,13 @@
 //! [Abstract Syntax Tree]: https://en.wikipedia.org/wiki/Abstract_syntax_tree
 //! [`Mode`]: crate::mode
 
-use crate::lexer::{LexResult, LexicalError, LexicalErrorType, Tok};
-pub use crate::mode::Mode;
-use crate::{ast, lexer, python};
-use ast::Location;
+use crate::{
+    ast::{self, Location},
+    lexer::{self, LexResult, LexicalError, LexicalErrorType},
+    mode::Mode,
+    python,
+    token::Tok,
+};
 use itertools::Itertools;
 use std::iter;
 
@@ -31,7 +34,7 @@ pub(super) use lalrpop_util::ParseError as LalrpopError;
 /// For example, parsing a simple function definition and a call to that function:
 ///
 /// ```
-/// use rustpython_parser::parser;
+/// use rustpython_parser as parser;
 /// let source = r#"
 /// def foo():
 ///    return 42
@@ -59,7 +62,7 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
 ///
 ///  ```
 /// extern crate num_bigint;
-/// use rustpython_parser::{parser, ast};
+/// use rustpython_parser as parser;
 /// let expr = parser::parse_expression("1 + 2", "<embedded>");
 ///
 /// assert!(expr.is_ok());
@@ -80,8 +83,7 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
 /// somewhat silly, location:
 ///
 /// ```
-/// use rustpython_parser::parser::parse_expression_located;
-/// use rustpython_parser::ast::Location;
+/// use rustpython_parser::{ast::Location, parse_expression_located};
 ///
 /// let expr = parse_expression_located("1 + 2", "<embedded>", Location::new(5, 20));
 /// assert!(expr.is_ok());
@@ -108,8 +110,7 @@ pub fn parse_expression_located(
 /// parsing:
 ///
 /// ```
-/// use rustpython_parser::mode::Mode;
-/// use rustpython_parser::parser::parse;
+/// use rustpython_parser::{Mode, parse};
 ///
 /// let expr = parse("1 + 2", Mode::Expression, "<embedded>");
 /// assert!(expr.is_ok());
@@ -118,8 +119,7 @@ pub fn parse_expression_located(
 /// Alternatively, we can parse a full Python program consisting of multiple lines:
 ///
 /// ```
-/// use rustpython_parser::mode::Mode;
-/// use rustpython_parser::parser::parse;
+/// use rustpython_parser::{Mode, parse};
 ///
 /// let source = r#"
 /// class Greeter:
@@ -142,9 +142,7 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
 /// # Example
 ///
 /// ```
-/// use rustpython_parser::ast::Location;
-/// use rustpython_parser::mode::Mode;
-/// use rustpython_parser::parser::parse_located;
+/// use rustpython_parser::{ast::Location, Mode, parse_located};
 ///
 /// let source = r#"
 /// def fib(i):
@@ -178,9 +176,7 @@ pub fn parse_located(
 /// them using the [`lexer::lex`] function:
 ///
 /// ```
-/// use rustpython_parser::lexer::lex;
-/// use rustpython_parser::mode::Mode;
-/// use rustpython_parser::parser::parse_tokens;
+/// use rustpython_parser::{lexer::lex, Mode, parse_tokens};
 ///
 /// let expr = parse_tokens(lex("1 + 2", Mode::Expression), Mode::Expression, "<embedded>");
 /// assert!(expr.is_ok());
@@ -200,9 +196,7 @@ pub fn parse_tokens(
 }
 
 /// Represents represent errors that occur during parsing and are
-/// returned by the `parse_*` functions in the [parser] module.
-///
-/// [parser]: crate::parser
+/// returned by the `parse_*` functions.
 pub type ParseError = rustpython_compiler_core::BaseError<ParseErrorType>;
 
 /// Represents the different types of errors that can occur during parsing.
diff --git a/parser/src/soft_keywords.rs b/parser/src/soft_keywords.rs
index 7011f3a7..2613914f 100644
--- a/parser/src/soft_keywords.rs
+++ b/parser/src/soft_keywords.rs
@@ -1,8 +1,6 @@
+use crate::{lexer::LexResult, mode::Mode, token::Tok};
 use itertools::{Itertools, MultiPeek};
 
-use crate::lexer::{LexResult, Tok};
-pub use crate::mode::Mode;
-
 /// An [`Iterator`] that transforms a token stream to accommodate soft keywords (namely, `match`
 /// and `case`).
 ///
diff --git a/parser/src/string.rs b/parser/src/string.rs
index 01431777..a8be7778 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -3,7 +3,6 @@
 // The lexer doesn't do any special handling of f-strings, it just treats them as
 // regular strings. Since the parser has no definition of f-string formats (Pending PEP 701)
 // we have to do the parsing here, manually.
-use self::FStringErrorType::*;
 use crate::{
     ast::{Constant, ConversionFlag, Expr, ExprKind, Location},
     lexer::{LexicalError, LexicalErrorType},
@@ -11,13 +10,12 @@ use crate::{
     token::{StringKind, Tok},
 };
 use itertools::Itertools;
-use std::{iter, str};
 
 // unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
 const MAX_UNICODE_NAME: usize = 88;
 
 struct StringParser<'a> {
-    chars: iter::Peekable<str::Chars<'a>>,
+    chars: std::iter::Peekable<std::str::Chars<'a>>,
     kind: StringKind,
     start: Location,
     end: Location,
@@ -177,6 +175,8 @@ impl<'a> StringParser<'a> {
     }
 
     fn parse_formatted_value(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
+        use FStringErrorType::*;
+
         let mut expression = String::new();
         let mut spec = None;
         let mut delims = Vec::new();
@@ -402,6 +402,8 @@ impl<'a> StringParser<'a> {
     }
 
     fn parse_fstring(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> {
+        use FStringErrorType::*;
+
         if nested >= 2 {
             return Err(FStringError::new(ExpressionNestedTooDeeply, self.get_pos()).into());
         }
@@ -653,7 +655,7 @@ pub(crate) fn parse_strings(
 // TODO: consolidate these with ParseError
 /// An error that occurred during parsing of an f-string.
 #[derive(Debug, PartialEq)]
-pub struct FStringError {
+struct FStringError {
     /// The type of error that occurred.
     pub error: FStringErrorType,
     /// The location of the error.
@@ -708,28 +710,29 @@ pub enum FStringErrorType {
 
 impl std::fmt::Display for FStringErrorType {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        use FStringErrorType::*;
         match self {
-            FStringErrorType::UnclosedLbrace => write!(f, "expecting '}}'"),
-            FStringErrorType::UnopenedRbrace => write!(f, "Unopened '}}'"),
-            FStringErrorType::ExpectedRbrace => write!(f, "Expected '}}' after conversion flag."),
-            FStringErrorType::InvalidExpression(error) => {
+            UnclosedLbrace => write!(f, "expecting '}}'"),
+            UnopenedRbrace => write!(f, "Unopened '}}'"),
+            ExpectedRbrace => write!(f, "Expected '}}' after conversion flag."),
+            InvalidExpression(error) => {
                 write!(f, "{error}")
             }
-            FStringErrorType::InvalidConversionFlag => write!(f, "invalid conversion character"),
-            FStringErrorType::EmptyExpression => write!(f, "empty expression not allowed"),
-            FStringErrorType::MismatchedDelimiter(first, second) => write!(
+            InvalidConversionFlag => write!(f, "invalid conversion character"),
+            EmptyExpression => write!(f, "empty expression not allowed"),
+            MismatchedDelimiter(first, second) => write!(
                 f,
                 "closing parenthesis '{second}' does not match opening parenthesis '{first}'"
             ),
-            FStringErrorType::SingleRbrace => write!(f, "single '}}' is not allowed"),
-            FStringErrorType::Unmatched(delim) => write!(f, "unmatched '{delim}'"),
-            FStringErrorType::ExpressionNestedTooDeeply => {
+            SingleRbrace => write!(f, "single '}}' is not allowed"),
+            Unmatched(delim) => write!(f, "unmatched '{delim}'"),
+            ExpressionNestedTooDeeply => {
                 write!(f, "expressions nested too deeply")
             }
-            FStringErrorType::UnterminatedString => {
+            UnterminatedString => {
                 write!(f, "unterminated string")
             }
-            FStringErrorType::ExpressionCannotInclude(c) => {
+            ExpressionCannotInclude(c) => {
                 if *c == '\\' {
                     write!(f, "f-string expression part cannot include a backslash")
                 } else {
@@ -832,6 +835,7 @@ mod tests {
 
     #[test]
     fn test_parse_invalid_fstring() {
+        use FStringErrorType::*;
         assert_eq!(parse_fstring_error("{5!a"), UnclosedLbrace);
         assert_eq!(parse_fstring_error("{5!a1}"), UnclosedLbrace);
         assert_eq!(parse_fstring_error("{5!"), UnclosedLbrace);
diff --git a/src/lib.rs b/src/lib.rs
index 275815d9..804ce10e 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,8 +1,7 @@
 use rustpython_codegen::{compile, symboltable};
 use rustpython_parser::{
+    self as parser,
     ast::{fold::Fold, ConstantOptimizer},
-    parser,
-    parser::ParseErrorType,
 };
 
 pub use rustpython_codegen::compile::CompileOpts;
@@ -13,13 +12,13 @@ pub enum CompileErrorType {
     #[error(transparent)]
     Codegen(#[from] rustpython_codegen::error::CodegenErrorType),
     #[error(transparent)]
-    Parse(#[from] rustpython_parser::parser::ParseErrorType),
+    Parse(#[from] parser::ParseErrorType),
 }
 
 pub type CompileError = rustpython_compiler_core::CompileError<CompileErrorType>;
 
-fn error_from_parse(error: rustpython_parser::parser::ParseError, source: &str) -> CompileError {
-    let error: CompileErrorBody<ParseErrorType> = error.into();
+fn error_from_parse(error: parser::ParseError, source: &str) -> CompileError {
+    let error: CompileErrorBody<parser::ParseErrorType> = error.into();
     CompileError::from(error, source)
 }
 

From 90da5ba54ab6fa00acc652ddf46e9d277f854671 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 22 Feb 2023 20:38:56 +0900
Subject: [PATCH 432/459] remove #[macro_use]

---
 parser/src/lexer.rs | 1 +
 parser/src/lib.rs   | 2 --
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index f4862b3a..45f5ca8b 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -38,6 +38,7 @@ use crate::{
     string::FStringErrorType,
     token::{StringKind, Tok},
 };
+use log::trace;
 use num_bigint::BigInt;
 use num_traits::{Num, Zero};
 use std::{char, cmp::Ordering, ops::Index, slice::SliceIndex, str::FromStr};
diff --git a/parser/src/lib.rs b/parser/src/lib.rs
index bc970740..9516932b 100644
--- a/parser/src/lib.rs
+++ b/parser/src/lib.rs
@@ -112,8 +112,6 @@
 #![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
 #![doc(html_root_url = "https://docs.rs/rustpython-parser/")]
 
-#[macro_use]
-extern crate log;
 pub use rustpython_ast as ast;
 
 mod function;

From 30f85787a75d4bfc7937b97e5b0491546c53e7a1 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 22 Feb 2023 21:01:39 +0900
Subject: [PATCH 433/459] use super::* from tests submodules

---
 ast/src/constant.rs    | 6 +++---
 codegen/src/compile.rs | 4 +---
 core/src/location.rs   | 2 +-
 parser/src/function.rs | 2 +-
 4 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/ast/src/constant.rs b/ast/src/constant.rs
index 55ee09a8..6099272e 100644
--- a/ast/src/constant.rs
+++ b/ast/src/constant.rs
@@ -133,12 +133,12 @@ impl<U> crate::fold::Fold<U> for ConstantOptimizer {
 
 #[cfg(test)]
 mod tests {
+    use super::*;
+
     #[cfg(feature = "constant-optimization")]
     #[test]
     fn test_constant_opt() {
-        use super::*;
-        use crate::fold::Fold;
-        use crate::*;
+        use crate::{fold::Fold, *};
 
         let start = Default::default();
         let end = None;
diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 92ed523c..252aff7d 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -2843,9 +2843,7 @@ fn compile_constant(value: &ast::Constant) -> ConstantData {
 
 #[cfg(test)]
 mod tests {
-    use super::{CompileOpts, Compiler};
-    use crate::symboltable::SymbolTable;
-    use rustpython_compiler_core::CodeObject;
+    use super::*;
     use rustpython_parser as parser;
 
     fn compile_exec(source: &str) -> CodeObject {
diff --git a/core/src/location.rs b/core/src/location.rs
index 9d32b54a..d9cd59be 100644
--- a/core/src/location.rs
+++ b/core/src/location.rs
@@ -96,7 +96,7 @@ impl Location {
 
 #[cfg(test)]
 mod tests {
-    use crate::Location;
+    use super::*;
 
     #[test]
     fn test_gt() {
diff --git a/parser/src/function.rs b/parser/src/function.rs
index cfda1c51..0f580e77 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -151,7 +151,7 @@ fn is_starred(exp: &ast::Expr) -> bool {
 
 #[cfg(test)]
 mod tests {
-    use crate::lexer::LexicalErrorType;
+    use super::*;
     use crate::parser::{parse_program, ParseErrorType};
 
     macro_rules! function_and_lambda {

From ba9e7c960747de703977a8f3f560677c0cb7d0e3 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Wed, 22 Feb 2023 10:01:30 -0500
Subject: [PATCH 434/459] Allow trailing commas in MappingPattern

---
 parser/python.lalrpop                         |   6 +-
 parser/src/parser.rs                          |  32 +-
 ...stpython_parser__parser__tests__match.snap | 949 ++++++++++++++++++
 ...tpython_parser__parser__tests__patma.snap} |   0
 4 files changed, 983 insertions(+), 4 deletions(-)
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__match.snap
 rename parser/src/snapshots/{rustpython_parser__parser__tests__match_complex.snap => rustpython_parser__parser__tests__patma.snap} (100%)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 9b3d226a..9dbc1ead 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -801,7 +801,7 @@ MappingPattern: ast::PatternKind = {
             rest: None,
         };
     },
-    <location:@L> "{" <e:OneOrMore<MatchMappingEntry>> "}" <end_location:@R> => {
+    <location:@L> "{" <e:OneOrMore<MatchMappingEntry>> ","? "}" <end_location:@R> => {
         let (keys, patterns) = e
             .into_iter()
             .unzip();
@@ -811,14 +811,14 @@ MappingPattern: ast::PatternKind = {
             rest: None,
         };
     },
-    <location:@L> "{" "**" <rest:Identifier> "}" <end_location:@R> => {
+    <location:@L> "{" "**" <rest:Identifier> ","? "}" <end_location:@R> => {
         return ast::PatternKind::MatchMapping {
             keys: vec![],
             patterns: vec![],
             rest: Some(rest),
         };
     },
-    <location:@L> "{" <e:OneOrMore<MatchMappingEntry>> ","  "**" <rest:Identifier> "}" <end_location:@R> => {
+    <location:@L> "{" <e:OneOrMore<MatchMappingEntry>> ","  "**" <rest:Identifier> ","? "}" <end_location:@R> => {
         let (keys, patterns) = e
             .into_iter()
             .unzip();
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index e0865b2d..9fe55707 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -497,7 +497,7 @@ match match:
     }
 
     #[test]
-    fn test_match_complex() {
+    fn test_patma() {
         let source = r#"# Cases sampled from Lib/test/test_patma.py
 
 # case test_patma_098
@@ -666,4 +666,34 @@ match w := x,:
         let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
+
+    #[test]
+    fn test_match() {
+        let parse_ast = parse_program(
+            r#"
+match {"test": 1}:
+    case {
+        **rest,
+    }:
+        print(rest)
+match {"label": "test"}:
+    case {
+        "label": str() | None as label,
+    }:
+        print(label)
+match x:
+    case [0, 1,]:
+        y = 0
+match x:
+    case (0, 1,):
+        y = 0
+match x:
+    case (0,):
+        y = 0
+"#,
+            "<test>",
+        )
+        .unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
 }
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__match.snap b/parser/src/snapshots/rustpython_parser__parser__tests__match.snap
new file mode 100644
index 00000000..421fad1e
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__match.snap
@@ -0,0 +1,949 @@
+---
+source: compiler/parser/src/parser.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 2,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 7,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 2,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 2,
+                        column: 17,
+                    },
+                ),
+                custom: (),
+                node: Dict {
+                    keys: [
+                        Some(
+                            Located {
+                                location: Location {
+                                    row: 2,
+                                    column: 7,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 2,
+                                        column: 13,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Str(
+                                        "test",
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        ),
+                    ],
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 2,
+                                column: 15,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 16,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Int(
+                                    1,
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 3,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 5,
+                                column: 5,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchMapping {
+                            keys: [],
+                            patterns: [],
+                            rest: Some(
+                                "rest",
+                            ),
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 6,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 6,
+                                    column: 19,
+                                },
+                            ),
+                            custom: (),
+                            node: Expr {
+                                value: Located {
+                                    location: Location {
+                                        row: 6,
+                                        column: 8,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 6,
+                                            column: 19,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Call {
+                                        func: Located {
+                                            location: Location {
+                                                row: 6,
+                                                column: 8,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 6,
+                                                    column: 13,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "print",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        args: [
+                                            Located {
+                                                location: Location {
+                                                    row: 6,
+                                                    column: 14,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 6,
+                                                        column: 18,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Name {
+                                                    id: "rest",
+                                                    ctx: Load,
+                                                },
+                                            },
+                                        ],
+                                        keywords: [],
+                                    },
+                                },
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 7,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 12,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 7,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 7,
+                        column: 23,
+                    },
+                ),
+                custom: (),
+                node: Dict {
+                    keys: [
+                        Some(
+                            Located {
+                                location: Location {
+                                    row: 7,
+                                    column: 7,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 7,
+                                        column: 14,
+                                    },
+                                ),
+                                custom: (),
+                                node: Constant {
+                                    value: Str(
+                                        "label",
+                                    ),
+                                    kind: None,
+                                },
+                            },
+                        ),
+                    ],
+                    values: [
+                        Located {
+                            location: Location {
+                                row: 7,
+                                column: 16,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 7,
+                                    column: 22,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Str(
+                                    "test",
+                                ),
+                                kind: None,
+                            },
+                        },
+                    ],
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 8,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 10,
+                                column: 5,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchMapping {
+                            keys: [
+                                Located {
+                                    location: Location {
+                                        row: 9,
+                                        column: 8,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 9,
+                                            column: 15,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Str(
+                                            "label",
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                            ],
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 9,
+                                        column: 17,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 9,
+                                            column: 38,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchAs {
+                                        pattern: Some(
+                                            Located {
+                                                location: Location {
+                                                    row: 9,
+                                                    column: 17,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 9,
+                                                        column: 29,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: MatchOr {
+                                                    patterns: [
+                                                        Located {
+                                                            location: Location {
+                                                                row: 9,
+                                                                column: 17,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 9,
+                                                                    column: 22,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchClass {
+                                                                cls: Located {
+                                                                    location: Location {
+                                                                        row: 9,
+                                                                        column: 17,
+                                                                    },
+                                                                    end_location: Some(
+                                                                        Location {
+                                                                            row: 9,
+                                                                            column: 20,
+                                                                        },
+                                                                    ),
+                                                                    custom: (),
+                                                                    node: Name {
+                                                                        id: "str",
+                                                                        ctx: Load,
+                                                                    },
+                                                                },
+                                                                patterns: [],
+                                                                kwd_attrs: [],
+                                                                kwd_patterns: [],
+                                                            },
+                                                        },
+                                                        Located {
+                                                            location: Location {
+                                                                row: 9,
+                                                                column: 25,
+                                                            },
+                                                            end_location: Some(
+                                                                Location {
+                                                                    row: 9,
+                                                                    column: 29,
+                                                                },
+                                                            ),
+                                                            custom: (),
+                                                            node: MatchSingleton {
+                                                                value: None,
+                                                            },
+                                                        },
+                                                    ],
+                                                },
+                                            },
+                                        ),
+                                        name: Some(
+                                            "label",
+                                        ),
+                                    },
+                                },
+                            ],
+                            rest: None,
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 11,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 11,
+                                    column: 20,
+                                },
+                            ),
+                            custom: (),
+                            node: Expr {
+                                value: Located {
+                                    location: Location {
+                                        row: 11,
+                                        column: 8,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 11,
+                                            column: 20,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Call {
+                                        func: Located {
+                                            location: Location {
+                                                row: 11,
+                                                column: 8,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 11,
+                                                    column: 13,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "print",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        args: [
+                                            Located {
+                                                location: Location {
+                                                    row: 11,
+                                                    column: 14,
+                                                },
+                                                end_location: Some(
+                                                    Location {
+                                                        row: 11,
+                                                        column: 19,
+                                                    },
+                                                ),
+                                                custom: (),
+                                                node: Name {
+                                                    id: "label",
+                                                    ctx: Load,
+                                                },
+                                            },
+                                        ],
+                                        keywords: [],
+                                    },
+                                },
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 12,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 15,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 12,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 12,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 13,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 13,
+                                column: 16,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 13,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 13,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 13,
+                                                column: 10,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 13,
+                                                    column: 11,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 13,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 13,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 13,
+                                                column: 13,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 13,
+                                                    column: 14,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    1,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 14,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 14,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 14,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 14,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 14,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 14,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 15,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 18,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 15,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 15,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 16,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 16,
+                                column: 16,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 16,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 16,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 16,
+                                                column: 10,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 16,
+                                                    column: 11,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                                Located {
+                                    location: Location {
+                                        row: 16,
+                                        column: 13,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 16,
+                                            column: 14,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 16,
+                                                column: 13,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 16,
+                                                    column: 14,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    1,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 17,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 17,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 17,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 17,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 17,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 17,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+    Located {
+        location: Location {
+            row: 18,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 21,
+                column: 0,
+            },
+        ),
+        custom: (),
+        node: Match {
+            subject: Located {
+                location: Location {
+                    row: 18,
+                    column: 6,
+                },
+                end_location: Some(
+                    Location {
+                        row: 18,
+                        column: 7,
+                    },
+                ),
+                custom: (),
+                node: Name {
+                    id: "x",
+                    ctx: Load,
+                },
+            },
+            cases: [
+                MatchCase {
+                    pattern: Located {
+                        location: Location {
+                            row: 19,
+                            column: 9,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 19,
+                                column: 13,
+                            },
+                        ),
+                        custom: (),
+                        node: MatchSequence {
+                            patterns: [
+                                Located {
+                                    location: Location {
+                                        row: 19,
+                                        column: 10,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 19,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: MatchValue {
+                                        value: Located {
+                                            location: Location {
+                                                row: 19,
+                                                column: 10,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 19,
+                                                    column: 11,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Constant {
+                                                value: Int(
+                                                    0,
+                                                ),
+                                                kind: None,
+                                            },
+                                        },
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                    guard: None,
+                    body: [
+                        Located {
+                            location: Location {
+                                row: 20,
+                                column: 8,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 20,
+                                    column: 13,
+                                },
+                            ),
+                            custom: (),
+                            node: Assign {
+                                targets: [
+                                    Located {
+                                        location: Location {
+                                            row: 20,
+                                            column: 8,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 20,
+                                                column: 9,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Name {
+                                            id: "y",
+                                            ctx: Store,
+                                        },
+                                    },
+                                ],
+                                value: Located {
+                                    location: Location {
+                                        row: 20,
+                                        column: 12,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 20,
+                                            column: 13,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Constant {
+                                        value: Int(
+                                            0,
+                                        ),
+                                        kind: None,
+                                    },
+                                },
+                                type_comment: None,
+                            },
+                        },
+                    ],
+                },
+            ],
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__match_complex.snap b/parser/src/snapshots/rustpython_parser__parser__tests__patma.snap
similarity index 100%
rename from parser/src/snapshots/rustpython_parser__parser__tests__match_complex.snap
rename to parser/src/snapshots/rustpython_parser__parser__tests__patma.snap

From e17437d935d66f7b6e2e075ca202b70de09ae9ca Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Wed, 22 Feb 2023 11:25:50 -0500
Subject: [PATCH 435/459] Limit match range to end of last statement

---
 parser/python.lalrpop                         |  30 +++-
 ...stpython_parser__parser__tests__match.snap |  20 +--
 ...r__parser__tests__match_as_identifier.snap |   4 +-
 ...stpython_parser__parser__tests__patma.snap | 140 +++++++++---------
 4 files changed, 109 insertions(+), 85 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 9dbc1ead..36730776 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -359,7 +359,15 @@ CompoundStatement: ast::Stmt = {
 };
 
 MatchStatement: ast::Stmt = {
-    <location:@L> "match" <subject:TestOrStarNamedExpr> ":" "\n" Indent <cases:MatchCase+> Dedent <end_location:@R> => {
+    <location:@L> "match" <subject:TestOrStarNamedExpr> ":" "\n" Indent <cases:MatchCase+> Dedent => {
+       let end_location = cases
+            .last()
+            .unwrap()
+            .body
+            .last()
+            .unwrap()
+            .end_location
+            .unwrap();
         ast::Stmt {
             location,
             end_location: Some(end_location),
@@ -370,7 +378,15 @@ MatchStatement: ast::Stmt = {
             }
         }
     },
-    <location:@L> "match" <subject:TestOrStarNamedExpr> "," ":" "\n" Indent <cases:MatchCase+> Dedent <end_location:@R> => {
+    <location:@L> "match" <subject:TestOrStarNamedExpr> "," ":" "\n" Indent <cases:MatchCase+> Dedent => {
+       let end_location = cases
+            .last()
+            .unwrap()
+            .body
+            .last()
+            .unwrap()
+            .end_location
+            .unwrap();
         ast::Stmt {
             location,
             end_location: Some(end_location),
@@ -381,7 +397,15 @@ MatchStatement: ast::Stmt = {
             }
         }
     },
-    <location:@L> "match" <subject:TestOrStarNamedExpr> "," <subjects:OneOrMore<TestOrStarNamedExpr>> ","? ":" "\n" Indent <cases:MatchCase+> Dedent <end_location:@R> => {
+    <location:@L> "match" <subject:TestOrStarNamedExpr> "," <subjects:OneOrMore<TestOrStarNamedExpr>> ","? ":" "\n" Indent <cases:MatchCase+> Dedent => {
+        let end_location = cases
+            .last()
+            .unwrap()
+            .body
+            .last()
+            .unwrap()
+            .end_location
+            .unwrap();
         let mut subjects = subjects;
         subjects.insert(0, subject);
         ast::Stmt {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__match.snap b/parser/src/snapshots/rustpython_parser__parser__tests__match.snap
index 421fad1e..6bc9c16a 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__match.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__match.snap
@@ -10,8 +10,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 7,
-                column: 0,
+                row: 6,
+                column: 19,
             },
         ),
         custom: (),
@@ -178,8 +178,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 12,
-                column: 0,
+                row: 11,
+                column: 20,
             },
         ),
         custom: (),
@@ -453,8 +453,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 15,
-                column: 0,
+                row: 14,
+                column: 13,
             },
         ),
         custom: (),
@@ -631,8 +631,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 18,
-                column: 0,
+                row: 17,
+                column: 13,
             },
         ),
         custom: (),
@@ -809,8 +809,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 21,
-                column: 0,
+                row: 20,
+                column: 13,
             },
         ),
         custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap b/parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap
index 2568cc78..4a3c8af3 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap
@@ -1538,8 +1538,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 21,
-                column: 0,
+                row: 20,
+                column: 12,
             },
         ),
         custom: (),
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__patma.snap b/parser/src/snapshots/rustpython_parser__parser__tests__patma.snap
index 83171725..8b7c8bec 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__patma.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__patma.snap
@@ -10,8 +10,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 8,
-                column: 0,
+                row: 6,
+                column: 13,
             },
         ),
         custom: (),
@@ -154,8 +154,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 12,
-                column: 0,
+                row: 10,
+                column: 13,
             },
         ),
         custom: (),
@@ -302,8 +302,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 18,
-                column: 0,
+                row: 16,
+                column: 13,
             },
         ),
         custom: (),
@@ -562,8 +562,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 22,
-                column: 0,
+                row: 20,
+                column: 16,
             },
         ),
         custom: (),
@@ -810,8 +810,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 26,
-                column: 0,
+                row: 24,
+                column: 13,
             },
         ),
         custom: (),
@@ -1090,8 +1090,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 32,
-                column: 0,
+                row: 30,
+                column: 20,
             },
         ),
         custom: (),
@@ -1257,8 +1257,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 40,
-                column: 0,
+                row: 38,
+                column: 13,
             },
         ),
         custom: (),
@@ -2102,8 +2102,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 44,
-                column: 0,
+                row: 42,
+                column: 13,
             },
         ),
         custom: (),
@@ -2265,8 +2265,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 48,
-                column: 0,
+                row: 46,
+                column: 13,
             },
         ),
         custom: (),
@@ -2409,8 +2409,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 52,
-                column: 0,
+                row: 50,
+                column: 16,
             },
         ),
         custom: (),
@@ -2657,8 +2657,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 56,
-                column: 0,
+                row: 54,
+                column: 13,
             },
         ),
         custom: (),
@@ -2802,8 +2802,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 64,
-                column: 0,
+                row: 62,
+                column: 13,
             },
         ),
         custom: (),
@@ -3179,8 +3179,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 68,
-                column: 0,
+                row: 66,
+                column: 13,
             },
         ),
         custom: (),
@@ -3322,8 +3322,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 74,
-                column: 0,
+                row: 72,
+                column: 13,
             },
         ),
         custom: (),
@@ -3542,8 +3542,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 78,
-                column: 0,
+                row: 76,
+                column: 15,
             },
         ),
         custom: (),
@@ -3691,8 +3691,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 82,
-                column: 0,
+                row: 80,
+                column: 13,
             },
         ),
         custom: (),
@@ -3979,8 +3979,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 90,
-                column: 0,
+                row: 88,
+                column: 13,
             },
         ),
         custom: (),
@@ -4515,8 +4515,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 94,
-                column: 0,
+                row: 92,
+                column: 13,
             },
         ),
         custom: (),
@@ -4679,8 +4679,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 98,
-                column: 0,
+                row: 96,
+                column: 13,
             },
         ),
         custom: (),
@@ -4858,8 +4858,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 102,
-                column: 0,
+                row: 100,
+                column: 13,
             },
         ),
         custom: (),
@@ -5005,8 +5005,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 106,
-                column: 0,
+                row: 104,
+                column: 13,
             },
         ),
         custom: (),
@@ -5181,8 +5181,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 110,
-                column: 0,
+                row: 108,
+                column: 13,
             },
         ),
         custom: (),
@@ -5290,8 +5290,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 114,
-                column: 0,
+                row: 112,
+                column: 13,
             },
         ),
         custom: (),
@@ -5417,8 +5417,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 118,
-                column: 0,
+                row: 116,
+                column: 13,
             },
         ),
         custom: (),
@@ -5528,8 +5528,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 126,
-                column: 0,
+                row: 124,
+                column: 13,
             },
         ),
         custom: (),
@@ -5840,8 +5840,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 130,
-                column: 0,
+                row: 128,
+                column: 13,
             },
         ),
         custom: (),
@@ -5952,8 +5952,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 134,
-                column: 0,
+                row: 132,
+                column: 13,
             },
         ),
         custom: (),
@@ -6118,8 +6118,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 138,
-                column: 0,
+                row: 136,
+                column: 13,
             },
         ),
         custom: (),
@@ -6480,8 +6480,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 146,
-                column: 0,
+                row: 144,
+                column: 13,
             },
         ),
         custom: (),
@@ -7325,8 +7325,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 150,
-                column: 0,
+                row: 148,
+                column: 13,
             },
         ),
         custom: (),
@@ -7545,8 +7545,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 154,
-                column: 0,
+                row: 152,
+                column: 13,
             },
         ),
         custom: (),
@@ -7765,8 +7765,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 158,
-                column: 0,
+                row: 156,
+                column: 13,
             },
         ),
         custom: (),
@@ -7894,8 +7894,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 162,
-                column: 0,
+                row: 160,
+                column: 13,
             },
         ),
         custom: (),
@@ -7907,8 +7907,8 @@ expression: parse_ast
                 },
                 end_location: Some(
                     Location {
-                        row: 162,
-                        column: 0,
+                        row: 160,
+                        column: 13,
                     },
                 ),
                 custom: (),
@@ -8077,8 +8077,8 @@ expression: parse_ast
         },
         end_location: Some(
             Location {
-                row: 165,
-                column: 0,
+                row: 164,
+                column: 13,
             },
         ),
         custom: (),

From 8b9f57906a241d836b9844561a95b9674ecfb20a Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Thu, 23 Feb 2023 08:37:44 -0500
Subject: [PATCH 436/459] Allow type variable tuple for *args

---
 parser/python.lalrpop                         |  21 +-
 parser/src/parser.rs                          |  12 ++
 ...ser__parser__tests__variadic_generics.snap | 188 ++++++++++++++++++
 3 files changed, 214 insertions(+), 7 deletions(-)
 create mode 100644 parser/src/snapshots/rustpython_parser__parser__tests__variadic_generics.snap

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index ee7f2970..4eceb5b0 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -1180,7 +1180,7 @@ FuncDef: ast::Stmt = {
 };
 
 Parameters: ast::Arguments = {
-    "(" <a: (ParameterList<TypedParameter>)?> ")" =>? {
+    "(" <a: (ParameterList<TypedParameter, StarTypedParameter>)?> ")" =>? {
         let args = validate_arguments(
             a.unwrap_or_else(|| ast::Arguments {
                 posonlyargs: vec![],
@@ -1199,8 +1199,8 @@ Parameters: ast::Arguments = {
 
 // Note that this is a macro which is used once for function defs, and
 // once for lambda defs.
-ParameterList<ArgType>: ast::Arguments = {
-    <param1:ParameterDefs<ArgType>> <args2:("," ParameterListStarArgs<ArgType>)?> ","? =>? {
+ParameterList<ArgType, StarArgType>: ast::Arguments = {
+    <param1:ParameterDefs<ArgType>> <args2:("," ParameterListStarArgs<ArgType, StarArgType>)?> ","? =>? {
         let (posonlyargs, args, defaults) = parse_params(param1)?;
 
         // Now gather rest of parameters:
@@ -1235,7 +1235,7 @@ ParameterList<ArgType>: ast::Arguments = {
             kw_defaults,
         })
     },
-    <params:ParameterListStarArgs<ArgType>> ","? => {
+    <params:ParameterListStarArgs<ArgType, StarArgType>> ","? => {
         let (vararg, kwonlyargs, kw_defaults, kwarg) = params;
         ast::Arguments {
             posonlyargs: vec![],
@@ -1291,11 +1291,18 @@ TypedParameter: ast::Arg = {
     },
 };
 
+StarTypedParameter: ast::Arg = {
+    <location:@L> <arg:Identifier> <a:(":" TestOrStarExpr)?> <end_location:@R> => {
+        let annotation = a.map(|x| Box::new(x.1));
+        ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None })
+    },
+};
+
 // Use inline here to make sure the "," is not creating an ambiguity.
 // TODO: figure out another grammar that makes this inline no longer required.
 #[inline]
-ParameterListStarArgs<ArgType>: (Option<Box<ast::Arg>>, Vec<ast::Arg>, Vec<ast::Expr>, Option<Box<ast::Arg>>) = {
-    <location:@L> "*" <va:ArgType?> <kw:("," ParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> =>? {
+ParameterListStarArgs<ArgType, StarArgType>: (Option<Box<ast::Arg>>, Vec<ast::Arg>, Vec<ast::Expr>, Option<Box<ast::Arg>>) = {
+    <location:@L> "*" <va:StarArgType?> <kw:("," ParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> =>? {
         // Extract keyword arguments:
         let mut kwonlyargs = Vec::new();
         let mut kw_defaults = Vec::new();
@@ -1413,7 +1420,7 @@ NamedExpression: ast::Expr = {
 };
 
 LambdaDef: ast::Expr = {
-    <location:@L> "lambda" <p:ParameterList<UntypedParameter>?> ":" <body:Test<"all">> <end_location:@R> =>? {
+    <location:@L> "lambda" <p:ParameterList<UntypedParameter, UntypedParameter>?> ":" <body:Test<"all">> <end_location:@R> =>? {
         let p = validate_arguments(
             p.unwrap_or_else(|| {
                 ast::Arguments {
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 50c705fa..c2c4a383 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -803,6 +803,18 @@ match x:
 match x:
     case (0,):
         y = 0
+"#,
+            "<test>",
+        )
+        .unwrap();
+        insta::assert_debug_snapshot!(parse_ast);
+    }
+
+    #[test]
+    fn test_variadic_generics() {
+        let parse_ast = parse_program(
+            r#"
+def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ...
 "#,
             "<test>",
         )
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__variadic_generics.snap b/parser/src/snapshots/rustpython_parser__parser__tests__variadic_generics.snap
new file mode 100644
index 00000000..a6875c3b
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__variadic_generics.snap
@@ -0,0 +1,188 @@
+---
+source: compiler/parser/src/parser.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 2,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 2,
+                column: 48,
+            },
+        ),
+        custom: (),
+        node: FunctionDef {
+            name: "args_to_tuple",
+            args: Arguments {
+                posonlyargs: [],
+                args: [],
+                vararg: Some(
+                    Located {
+                        location: Location {
+                            row: 2,
+                            column: 19,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 2,
+                                column: 28,
+                            },
+                        ),
+                        custom: (),
+                        node: ArgData {
+                            arg: "args",
+                            annotation: Some(
+                                Located {
+                                    location: Location {
+                                        row: 2,
+                                        column: 25,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 2,
+                                            column: 28,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Starred {
+                                        value: Located {
+                                            location: Location {
+                                                row: 2,
+                                                column: 26,
+                                            },
+                                            end_location: Some(
+                                                Location {
+                                                    row: 2,
+                                                    column: 28,
+                                                },
+                                            ),
+                                            custom: (),
+                                            node: Name {
+                                                id: "Ts",
+                                                ctx: Load,
+                                            },
+                                        },
+                                        ctx: Load,
+                                    },
+                                },
+                            ),
+                            type_comment: None,
+                        },
+                    },
+                ),
+                kwonlyargs: [],
+                kw_defaults: [],
+                kwarg: None,
+                defaults: [],
+            },
+            body: [
+                Located {
+                    location: Location {
+                        row: 2,
+                        column: 45,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 2,
+                            column: 48,
+                        },
+                    ),
+                    custom: (),
+                    node: Expr {
+                        value: Located {
+                            location: Location {
+                                row: 2,
+                                column: 45,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 48,
+                                },
+                            ),
+                            custom: (),
+                            node: Constant {
+                                value: Ellipsis,
+                                kind: None,
+                            },
+                        },
+                    },
+                },
+            ],
+            decorator_list: [],
+            returns: Some(
+                Located {
+                    location: Location {
+                        row: 2,
+                        column: 33,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 2,
+                            column: 43,
+                        },
+                    ),
+                    custom: (),
+                    node: Subscript {
+                        value: Located {
+                            location: Location {
+                                row: 2,
+                                column: 33,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 38,
+                                },
+                            ),
+                            custom: (),
+                            node: Name {
+                                id: "Tuple",
+                                ctx: Load,
+                            },
+                        },
+                        slice: Located {
+                            location: Location {
+                                row: 2,
+                                column: 39,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 2,
+                                    column: 42,
+                                },
+                            ),
+                            custom: (),
+                            node: Starred {
+                                value: Located {
+                                    location: Location {
+                                        row: 2,
+                                        column: 40,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 2,
+                                            column: 42,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "Ts",
+                                        ctx: Load,
+                                    },
+                                },
+                                ctx: Load,
+                            },
+                        },
+                        ctx: Load,
+                    },
+                },
+            ),
+            type_comment: None,
+        },
+    },
+]

From ce5cdd1bcb5e19f029ba67d55c0c83a0251e3f9e Mon Sep 17 00:00:00 2001
From: DimitrisJim <d.f.hilliard@gmail.com>
Date: Sun, 26 Feb 2023 09:23:07 +0200
Subject: [PATCH 437/459] Always load __debug__ as a const.

---
 codegen/src/compile.rs | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 252aff7d..b61c7600 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -500,6 +500,14 @@ impl Compiler {
             // // TODO: is this right?
             // SymbolScope::Unknown => NameOpType::Global,
         };
+
+        if NameUsage::Load == usage && name == "__debug__" {
+            self.emit_constant(ConstantData::Boolean {
+                value: self.opts.optimize == 0,
+            });
+            return Ok(());
+        }
+
         let mut idx = cache
             .get_index_of(name.as_ref())
             .unwrap_or_else(|| cache.insert_full(name.into_owned()).0);

From fdea016ea7059354aff695a5b33237f13b98ae1c Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Sun, 26 Feb 2023 23:06:06 -0500
Subject: [PATCH 438/459] Use proper locations for sub-expressions in constant
 matches

---
 parser/python.lalrpop                         | 185 ++++--------------
 ...stpython_parser__parser__tests__patma.snap |  16 +-
 2 files changed, 50 insertions(+), 151 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 4eceb5b0..c8936d33 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -576,6 +576,41 @@ StarPattern: ast::PatternKind = {
     },
 }
 
+ConstantAtom: ast::Expr = {
+    <location:@L> <value:Constant> <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::Constant { value, kind: None }
+    },
+}
+
+ConstantExpr: ast::Expr = {
+    ConstantAtom,
+    <location:@L> "-" <operand:ConstantAtom> <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::UnaryOp {
+            op: ast::Unaryop::USub,
+            operand: Box::new(operand)
+        }
+    },
+}
+
+AddOpExpr: ast::Expr = {
+    <location:@L> <left:ConstantExpr> <op:AddOp> <right:ConstantAtom> <end_location:@R> => ast::Expr {
+        location,
+        end_location: Some(end_location),
+        custom: (),
+        node: ast::ExprKind::BinOp {
+            left: Box::new(left),
+            op,
+            right: Box::new(right),
+        }
+    },
+}
+
 LiteralPattern: ast::PatternKind = {
     "None" => ast::PatternKind::MatchSingleton {
         value: ast::Constant::None
@@ -586,81 +621,11 @@ LiteralPattern: ast::PatternKind = {
     "False" => ast::PatternKind::MatchSingleton {
         value: false.into()
     },
-    <location:@L> <value:Constant> <end_location:@R> => ast::PatternKind::MatchValue {
-        value: Box::new(ast::Expr {
-            location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::Constant { value, kind: None }
-        })
-    },
-    <location:@L> "-" <value:Constant> <end_location:@R> => ast::PatternKind::MatchValue {
-        value: Box::new(ast::Expr {
-            location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::UnaryOp {
-                op: ast::Unaryop::USub,
-                operand: Box::new(ast::Expr {
-                    location,
-                    end_location: Some(end_location),
-                    custom: (),
-                    node: ast::ExprKind::Constant { value, kind: None }
-                })
-            }
-        })
-    },
-    <location:@L> <left:Constant> <op:AddOp> <right:Constant> <end_location:@R> => ast::PatternKind::MatchValue {
-        value: Box::new(ast::Expr {
-            location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::BinOp {
-                left: Box::new(ast::Expr {
-                    location,
-                    end_location: Some(end_location),
-                    custom: (),
-                    node: ast::ExprKind::Constant { value: left, kind: None }
-                }),
-                op,
-                right: Box::new(ast::Expr {
-                    location,
-                    end_location: Some(end_location),
-                    custom: (),
-                    node: ast::ExprKind::Constant { value: right, kind: None }
-                }),
-            }
-        })
+    <location:@L> <value:ConstantExpr> <end_location:@R> => ast::PatternKind::MatchValue {
+        value: Box::new(value)
     },
-    <location:@L> "-" <left:Constant> <op:AddOp> <right:Constant> <end_location:@R> => ast::PatternKind::MatchValue {
-        value: Box::new(ast::Expr {
-            location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::BinOp {
-                left: Box::new(ast::Expr {
-                    location,
-                    end_location: Some(end_location),
-                    custom: (),
-                    node: ast::ExprKind::UnaryOp {
-                        op: ast::Unaryop::USub,
-                        operand: Box::new(ast::Expr {
-                            location,
-                            end_location: Some(end_location),
-                            custom: (),
-                            node: ast::ExprKind::Constant { value: left, kind: None }
-                        })
-                    }
-                }),
-                op,
-                right: Box::new(ast::Expr {
-                    location,
-                    end_location: Some(end_location),
-                    custom: (),
-                    node: ast::ExprKind::Constant { value: right, kind: None }
-                }),
-            }
-        })
+    <location:@L> <value:AddOpExpr> => ast::PatternKind::MatchValue {
+        value: Box::new(value)
     },
     <location:@L> <s:(@L string @R)+> =>? Ok(ast::PatternKind::MatchValue {
         value: Box::new(parse_strings(s)?)
@@ -713,6 +678,9 @@ ValuePattern: ast::PatternKind = {
 }
 
 MappingKey: ast::Expr = {
+    ConstantExpr,
+    AddOpExpr,
+    MatchNameOrAttr,
     <location:@L> "None" <end_location:@R> => ast::Expr {
         location,
         end_location: Some(end_location),
@@ -740,76 +708,7 @@ MappingKey: ast::Expr = {
             kind: None,
         },
     },
-    <location:@L> <value:Constant> <end_location:@R> => ast::Expr {
-        location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Constant { value, kind: None }
-    },
-    <location:@L> "-" <value:Constant> <end_location:@R> => ast::Expr {
-        location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::UnaryOp {
-            op: ast::Unaryop::USub,
-            operand: Box::new(ast::Expr {
-                location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::ExprKind::Constant { value, kind: None }
-            })
-        }
-    },
-    <location:@L> <left:Constant> <op:AddOp> <right:Constant> <end_location:@R> => ast::Expr {
-        location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::BinOp {
-            left: Box::new(ast::Expr {
-                location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::ExprKind::Constant { value: left, kind: None }
-            }),
-            op,
-            right: Box::new(ast::Expr {
-                location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::ExprKind::Constant { value: right, kind: None }
-            }),
-        }
-    },
-    <location:@L> "-" <left:Constant> <op:AddOp> <right:Constant> <end_location:@R> => ast::Expr {
-        location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::BinOp {
-            left: Box::new(ast::Expr {
-                location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::ExprKind::UnaryOp {
-                    op: ast::Unaryop::USub,
-                    operand: Box::new(ast::Expr {
-                        location,
-                        end_location: Some(end_location),
-                        custom: (),
-                        node: ast::ExprKind::Constant { value: left, kind: None }
-                    })
-                }
-            }),
-            op,
-            right: Box::new(ast::Expr {
-                location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::ExprKind::Constant { value: right, kind: None }
-            }),
-        }
-    },
     <location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?),
-    MatchNameOrAttr,
 }
 
 MatchMappingEntry: (ast::Expr, ast::Pattern) = {
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__patma.snap b/parser/src/snapshots/rustpython_parser__parser__tests__patma.snap
index 8b7c8bec..147ba242 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__patma.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__patma.snap
@@ -65,7 +65,7 @@ expression: parse_ast
                                     operand: Located {
                                         location: Location {
                                             row: 5,
-                                            column: 9,
+                                            column: 10,
                                         },
                                         end_location: Some(
                                             Location {
@@ -2161,7 +2161,7 @@ expression: parse_ast
                                         end_location: Some(
                                             Location {
                                                 row: 41,
-                                                column: 21,
+                                                column: 13,
                                             },
                                         ),
                                         custom: (),
@@ -2176,7 +2176,7 @@ expression: parse_ast
                                     right: Located {
                                         location: Location {
                                             row: 41,
-                                            column: 9,
+                                            column: 16,
                                         },
                                         end_location: Some(
                                             Location {
@@ -2320,7 +2320,7 @@ expression: parse_ast
                                     operand: Located {
                                         location: Location {
                                             row: 45,
-                                            column: 9,
+                                            column: 10,
                                         },
                                         end_location: Some(
                                             Location {
@@ -4738,7 +4738,7 @@ expression: parse_ast
                                         end_location: Some(
                                             Location {
                                                 row: 95,
-                                                column: 22,
+                                                column: 14,
                                             },
                                         ),
                                         custom: (),
@@ -4747,12 +4747,12 @@ expression: parse_ast
                                             operand: Located {
                                                 location: Location {
                                                     row: 95,
-                                                    column: 9,
+                                                    column: 10,
                                                 },
                                                 end_location: Some(
                                                     Location {
                                                         row: 95,
-                                                        column: 22,
+                                                        column: 14,
                                                     },
                                                 ),
                                                 custom: (),
@@ -4769,7 +4769,7 @@ expression: parse_ast
                                     right: Located {
                                         location: Location {
                                             row: 95,
-                                            column: 9,
+                                            column: 17,
                                         },
                                         end_location: Some(
                                             Location {

From 268f3ad004300a49b2e41cbfe2aeeba7e24687b2 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Sun, 26 Feb 2023 19:09:13 +0900
Subject: [PATCH 439/459] Use git version of unicode_names2 to avoid alias
 search failure

---
 parser/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index 56f3d161..40bf75e5 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -27,10 +27,10 @@ log = { workspace = true }
 num-bigint = { workspace = true }
 num-traits = { workspace = true }
 thiserror = { workspace = true }
+unicode_names2 = { workspace = true }
 
 unic-emoji-char = "0.9.0"
 unic-ucd-ident  = "0.9.0"
-unicode_names2 = "0.5.0"
 lalrpop-util = "0.19.8"
 phf = "0.11.1"
 rustc-hash = "1.1.0"

From 8cb2cca9fb655859533d771c912feae3244d9850 Mon Sep 17 00:00:00 2001
From: DimitrisJim <d.f.hilliard@gmail.com>
Date: Sat, 25 Feb 2023 13:35:22 +0200
Subject: [PATCH 440/459] Use insta to verify values.

---
 ...arser__string__tests__backspace_alias.snap | 40 +++++++++++++++++++
 ...hon_parser__string__tests__bell_alias.snap | 40 +++++++++++++++++++
 ..._string__tests__carriage_return_alias.snap | 40 +++++++++++++++++++
 ...r_tabulation_with_justification_alias.snap | 40 +++++++++++++++++++
 ...n_parser__string__tests__delete_alias.snap | 40 +++++++++++++++++++
 ...n_parser__string__tests__escape_alias.snap | 40 +++++++++++++++++++
 ...arser__string__tests__form_feed_alias.snap | 40 +++++++++++++++++++
 ...thon_parser__string__tests__hts_alias.snap | 40 +++++++++++++++++++
 parser/src/string.rs                          | 24 +++++++++++
 9 files changed, 344 insertions(+)
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__backspace_alias.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__bell_alias.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__carriage_return_alias.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__character_tabulation_with_justification_alias.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__delete_alias.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__escape_alias.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__form_feed_alias.snap
 create mode 100644 parser/src/snapshots/rustpython_parser__string__tests__hts_alias.snap

diff --git a/parser/src/snapshots/rustpython_parser__string__tests__backspace_alias.snap b/parser/src/snapshots/rustpython_parser__string__tests__backspace_alias.snap
new file mode 100644
index 00000000..eea97869
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__backspace_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 15,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 15,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{8}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__bell_alias.snap b/parser/src/snapshots/rustpython_parser__string__tests__bell_alias.snap
new file mode 100644
index 00000000..cf04fd68
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__bell_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 9,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 9,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{7}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__carriage_return_alias.snap b/parser/src/snapshots/rustpython_parser__string__tests__carriage_return_alias.snap
new file mode 100644
index 00000000..33a84d22
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__carriage_return_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 21,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 21,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\r",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__character_tabulation_with_justification_alias.snap b/parser/src/snapshots/rustpython_parser__string__tests__character_tabulation_with_justification_alias.snap
new file mode 100644
index 00000000..282706f8
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__character_tabulation_with_justification_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 45,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 45,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{89}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__delete_alias.snap b/parser/src/snapshots/rustpython_parser__string__tests__delete_alias.snap
new file mode 100644
index 00000000..06e74d24
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__delete_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 12,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 12,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{7f}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__escape_alias.snap b/parser/src/snapshots/rustpython_parser__string__tests__escape_alias.snap
new file mode 100644
index 00000000..751456db
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__escape_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 12,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 12,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{1b}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__form_feed_alias.snap b/parser/src/snapshots/rustpython_parser__string__tests__form_feed_alias.snap
new file mode 100644
index 00000000..b91c10eb
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__form_feed_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 15,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 15,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{c}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__hts_alias.snap b/parser/src/snapshots/rustpython_parser__string__tests__hts_alias.snap
new file mode 100644
index 00000000..73f12074
--- /dev/null
+++ b/parser/src/snapshots/rustpython_parser__string__tests__hts_alias.snap
@@ -0,0 +1,40 @@
+---
+source: compiler/parser/src/string.rs
+expression: parse_ast
+---
+[
+    Located {
+        location: Location {
+            row: 1,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 1,
+                column: 9,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 1,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 1,
+                        column: 9,
+                    },
+                ),
+                custom: (),
+                node: Constant {
+                    value: Str(
+                        "\u{88}",
+                    ),
+                    kind: None,
+                },
+            },
+        },
+    },
+]
diff --git a/parser/src/string.rs b/parser/src/string.rs
index a8be7778..0cd03df6 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -1048,4 +1048,28 @@ mod tests {
         let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
+
+    macro_rules! test_aliases_parse {
+        ($($name:ident: $alias:expr,)*) => {
+        $(
+            #[test]
+            fn $name() {
+                let source = format!(r#""\N{{{0}}}""#, $alias);
+                let parse_ast = parse_program(&source, "<test>").unwrap();
+                insta::assert_debug_snapshot!(parse_ast);
+            }
+        )*
+        }
+    }
+
+    test_aliases_parse! {
+        test_backspace_alias: "BACKSPACE",
+        test_bell_alias: "BEL",
+        test_carriage_return_alias: "CARRIAGE RETURN",
+        test_delete_alias: "DELETE",
+        test_escape_alias: "ESCAPE",
+        test_form_feed_alias: "FORM FEED",
+        test_hts_alias: "HTS",
+        test_character_tabulation_with_justification_alias: "CHARACTER TABULATION WITH JUSTIFICATION",
+    }
 }

From 50b5388711eb2caeb1a1fa6a8f3ad58e63de0109 Mon Sep 17 00:00:00 2001
From: DimitrisJim <d.f.hilliard@gmail.com>
Date: Wed, 1 Mar 2023 11:48:45 +0200
Subject: [PATCH 441/459] Disallow __debug__ as args, kwargs name.

---
 codegen/src/compile.rs | 36 +++++++++++++++++-------------------
 1 file changed, 17 insertions(+), 19 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index b61c7600..be1c97db 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -315,8 +315,13 @@ impl Compiler {
     fn name(&mut self, name: &str) -> bytecode::NameIdx {
         self._name_inner(name, |i| &mut i.name_cache)
     }
-    fn varname(&mut self, name: &str) -> bytecode::NameIdx {
-        self._name_inner(name, |i| &mut i.varname_cache)
+    fn varname(&mut self, name: &str) -> CompileResult<bytecode::NameIdx> {
+        if Compiler::is_forbidden_arg_name(name) {
+            return Err(self.error(CodegenErrorType::SyntaxError(format!(
+                "cannot assign to {name}",
+            ))));
+        }
+        Ok(self._name_inner(name, |i| &mut i.varname_cache))
     }
     fn _name_inner(
         &mut self,
@@ -942,24 +947,17 @@ impl Compiler {
             .chain(&args.args)
             .chain(&args.kwonlyargs);
         for name in args_iter {
-            if Compiler::is_forbidden_arg_name(&name.node.arg) {
-                return Err(self.error(CodegenErrorType::SyntaxError(format!(
-                    "cannot assign to {}",
-                    &name.node.arg
-                ))));
-            }
-            self.varname(&name.node.arg);
+            self.varname(&name.node.arg)?;
         }
 
-        let mut compile_varargs = |va: Option<&ast::Arg>, flag| {
-            if let Some(name) = va {
-                self.current_codeinfo().flags |= flag;
-                self.varname(&name.node.arg);
-            }
-        };
-
-        compile_varargs(args.vararg.as_deref(), bytecode::CodeFlags::HAS_VARARGS);
-        compile_varargs(args.kwarg.as_deref(), bytecode::CodeFlags::HAS_VARKEYWORDS);
+        if let Some(name) = args.vararg.as_deref() {
+            self.current_codeinfo().flags |= bytecode::CodeFlags::HAS_VARARGS;
+            self.varname(&name.node.arg)?;
+        }
+        if let Some(name) = args.kwarg.as_deref() {
+            self.current_codeinfo().flags |= bytecode::CodeFlags::HAS_VARKEYWORDS;
+            self.varname(&name.node.arg)?;
+        }
 
         Ok(funcflags)
     }
@@ -2565,7 +2563,7 @@ impl Compiler {
             0,
             name.to_owned(),
         );
-        let arg0 = self.varname(".0");
+        let arg0 = self.varname(".0")?;
 
         let return_none = init_collection.is_none();
         // Create empty object of proper type:

From 41b465dee132818d15a766462e40bad0ea7449f3 Mon Sep 17 00:00:00 2001
From: Noa <coolreader18@gmail.com>
Date: Thu, 15 Dec 2022 13:25:02 -0600
Subject: [PATCH 442/459] Custom marshal enc/decoding impl

---
 Cargo.toml             |   2 -
 codegen/Cargo.toml     |   3 +-
 codegen/src/compile.rs |  18 +-
 codegen/src/error.rs   |   4 +-
 codegen/src/ir.rs      |  18 +-
 core/Cargo.toml        |   8 +-
 core/src/bytecode.rs   | 408 ++++++++++++++-------------
 core/src/error.rs      |  18 +-
 core/src/lib.rs        |   1 +
 core/src/location.rs   |   4 +-
 core/src/marshal.rs    | 615 +++++++++++++++++++++++++++++++++++++++++
 parser/Cargo.toml      |   1 -
 parser/src/parser.rs   |   4 +-
 src/lib.rs             |  38 ++-
 14 files changed, 913 insertions(+), 229 deletions(-)
 create mode 100644 core/src/marshal.rs

diff --git a/Cargo.toml b/Cargo.toml
index b6e204df..ff6ea775 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,5 +9,3 @@ edition = "2021"
 rustpython-compiler-core = { path = "core" }
 rustpython-codegen = { path = "codegen" }
 rustpython-parser = { path = "parser" }
-
-thiserror = { workspace = true }
diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml
index fd6ad653..3bb891ed 100644
--- a/codegen/Cargo.toml
+++ b/codegen/Cargo.toml
@@ -16,9 +16,8 @@ bitflags = { workspace = true }
 indexmap = { workspace = true }
 itertools = { workspace = true }
 log = { workspace = true }
-num-complex = { workspace = true, features = ["serde"] }
+num-complex = { workspace = true }
 num-traits = { workspace = true }
-thiserror = { workspace = true }
 
 [dev-dependencies]
 rustpython-parser = { path = "../parser" }
diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index be1c97db..2e80de4e 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -249,9 +249,9 @@ impl Compiler {
     fn push_output(
         &mut self,
         flags: bytecode::CodeFlags,
-        posonlyarg_count: usize,
-        arg_count: usize,
-        kwonlyarg_count: usize,
+        posonlyarg_count: u32,
+        arg_count: u32,
+        kwonlyarg_count: u32,
         obj_name: String,
     ) {
         let source_path = self.source_path.clone();
@@ -936,9 +936,11 @@ impl Compiler {
 
         self.push_output(
             bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED,
-            args.posonlyargs.len(),
-            args.posonlyargs.len() + args.args.len(),
-            args.kwonlyargs.len(),
+            args.posonlyargs.len().try_into().unwrap(),
+            (args.posonlyargs.len() + args.args.len())
+                .try_into()
+                .unwrap(),
+            args.kwonlyargs.len().try_into().unwrap(),
             name.to_owned(),
         );
 
@@ -2750,8 +2752,8 @@ impl Compiler {
         self.current_source_location = location;
     }
 
-    fn get_source_line_number(&self) -> usize {
-        self.current_source_location.row()
+    fn get_source_line_number(&self) -> u32 {
+        self.current_source_location.row() as u32
     }
 
     fn push_qualified_path(&mut self, name: &str) {
diff --git a/codegen/src/error.rs b/codegen/src/error.rs
index 648446f4..e0380a62 100644
--- a/codegen/src/error.rs
+++ b/codegen/src/error.rs
@@ -2,7 +2,7 @@ use std::fmt;
 
 pub type CodegenError = rustpython_compiler_core::BaseError<CodegenErrorType>;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug)]
 #[non_exhaustive]
 pub enum CodegenErrorType {
     /// Invalid assignment, cannot store value in target.
@@ -33,6 +33,8 @@ pub enum CodegenErrorType {
     NotImplementedYet, // RustPython marker for unimplemented features
 }
 
+impl std::error::Error for CodegenErrorType {}
+
 impl fmt::Display for CodegenErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         use CodegenErrorType::*;
diff --git a/codegen/src/ir.rs b/codegen/src/ir.rs
index 43d7be0f..2658028d 100644
--- a/codegen/src/ir.rs
+++ b/codegen/src/ir.rs
@@ -63,11 +63,11 @@ impl Default for Block {
 
 pub struct CodeInfo {
     pub flags: CodeFlags,
-    pub posonlyarg_count: usize, // Number of positional-only arguments
-    pub arg_count: usize,
-    pub kwonlyarg_count: usize,
+    pub posonlyarg_count: u32, // Number of positional-only arguments
+    pub arg_count: u32,
+    pub kwonlyarg_count: u32,
     pub source_path: String,
-    pub first_line_number: usize,
+    pub first_line_number: u32,
     pub obj_name: String, // Name of the object that created this code object
 
     pub blocks: Vec<Block>,
@@ -172,15 +172,15 @@ impl CodeInfo {
         }
     }
 
-    fn cell2arg(&self) -> Option<Box<[isize]>> {
+    fn cell2arg(&self) -> Option<Box<[i32]>> {
         if self.cellvar_cache.is_empty() {
             return None;
         }
 
         let total_args = self.arg_count
             + self.kwonlyarg_count
-            + self.flags.contains(CodeFlags::HAS_VARARGS) as usize
-            + self.flags.contains(CodeFlags::HAS_VARKEYWORDS) as usize;
+            + self.flags.contains(CodeFlags::HAS_VARARGS) as u32
+            + self.flags.contains(CodeFlags::HAS_VARKEYWORDS) as u32;
 
         let mut found_cellarg = false;
         let cell2arg = self
@@ -190,10 +190,10 @@ impl CodeInfo {
                 self.varname_cache
                     .get_index_of(var)
                     // check that it's actually an arg
-                    .filter(|i| *i < total_args)
+                    .filter(|i| *i < total_args as usize)
                     .map_or(-1, |i| {
                         found_cellarg = true;
-                        i as isize
+                        i as i32
                     })
             })
             .collect::<Box<[_]>>();
diff --git a/core/Cargo.toml b/core/Cargo.toml
index bf385a71..03160356 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -8,14 +8,10 @@ repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
 [dependencies]
-bincode = { workspace = true }
 bitflags = { workspace = true }
 bstr = { workspace = true }
 itertools = { workspace = true }
-num-bigint = { workspace = true, features = ["serde"] }
-num-complex = { workspace = true, features = ["serde"] }
-num_enum = { workspace = true }
-serde = { workspace = true, features = ["derive"] }
-thiserror = { workspace = true }
+num-bigint = { workspace = true }
+num-complex = { workspace = true }
 
 lz4_flex = "0.9.2"
diff --git a/core/src/bytecode.rs b/core/src/bytecode.rs
index b7c456e9..fa7146d7 100644
--- a/core/src/bytecode.rs
+++ b/core/src/bytecode.rs
@@ -1,14 +1,12 @@
 //! Implement python as a virtual machine with bytecodes. This module
 //! implements bytecode structure.
 
-use crate::Location;
+use crate::marshal::MarshalError;
+use crate::{marshal, Location};
 use bitflags::bitflags;
-use bstr::ByteSlice;
 use itertools::Itertools;
 use num_bigint::BigInt;
 use num_complex::Complex64;
-use num_enum::{IntoPrimitive, TryFromPrimitive};
-use serde::{Deserialize, Serialize};
 use std::marker::PhantomData;
 use std::{collections::BTreeSet, fmt, hash, mem};
 
@@ -31,9 +29,7 @@ impl Constant for ConstantData {
             ConstantData::Str { value } => Str { value },
             ConstantData::Bytes { value } => Bytes { value },
             ConstantData::Code { code } => Code { code },
-            ConstantData::Tuple { elements } => Tuple {
-                elements: Box::new(elements.iter().map(|e| e.borrow_constant())),
-            },
+            ConstantData::Tuple { elements } => Tuple { elements },
             ConstantData::None => None,
             ConstantData::Ellipsis => Ellipsis,
         }
@@ -44,6 +40,9 @@ impl Constant for ConstantData {
 pub trait ConstantBag: Sized + Copy {
     type Constant: Constant;
     fn make_constant<C: Constant>(&self, constant: BorrowedConstant<C>) -> Self::Constant;
+    fn make_int(&self, value: BigInt) -> Self::Constant;
+    fn make_tuple(&self, elements: impl Iterator<Item = Self::Constant>) -> Self::Constant;
+    fn make_code(&self, code: CodeObject<Self::Constant>) -> Self::Constant;
     fn make_name(&self, name: &str) -> <Self::Constant as Constant>::Name;
 }
 
@@ -55,6 +54,19 @@ impl ConstantBag for BasicBag {
     fn make_constant<C: Constant>(&self, constant: BorrowedConstant<C>) -> Self::Constant {
         constant.to_owned()
     }
+    fn make_int(&self, value: BigInt) -> Self::Constant {
+        ConstantData::Integer { value }
+    }
+    fn make_tuple(&self, elements: impl Iterator<Item = Self::Constant>) -> Self::Constant {
+        ConstantData::Tuple {
+            elements: elements.collect(),
+        }
+    }
+    fn make_code(&self, code: CodeObject<Self::Constant>) -> Self::Constant {
+        ConstantData::Code {
+            code: Box::new(code),
+        }
+    }
     fn make_name(&self, name: &str) -> <Self::Constant as Constant>::Name {
         name.to_owned()
     }
@@ -62,26 +74,22 @@ impl ConstantBag for BasicBag {
 
 /// Primary container of a single code object. Each python function has
 /// a codeobject. Also a module has a codeobject.
-#[derive(Clone, Serialize, Deserialize)]
+#[derive(Clone)]
 pub struct CodeObject<C: Constant = ConstantData> {
     pub instructions: Box<[CodeUnit]>,
     pub locations: Box<[Location]>,
     pub flags: CodeFlags,
-    pub posonlyarg_count: usize,
+    pub posonlyarg_count: u32,
     // Number of positional-only arguments
-    pub arg_count: usize,
-    pub kwonlyarg_count: usize,
+    pub arg_count: u32,
+    pub kwonlyarg_count: u32,
     pub source_path: C::Name,
-    pub first_line_number: usize,
+    pub first_line_number: u32,
     pub max_stackdepth: u32,
     pub obj_name: C::Name,
     // Name of the object that created this code object
-    pub cell2arg: Option<Box<[isize]>>,
+    pub cell2arg: Option<Box<[i32]>>,
     pub constants: Box<[C]>,
-    #[serde(bound(
-        deserialize = "C::Name: serde::Deserialize<'de>",
-        serialize = "C::Name: serde::Serialize"
-    ))]
     pub names: Box<[C::Name]>,
     pub varnames: Box<[C::Name]>,
     pub cellvars: Box<[C::Name]>,
@@ -89,7 +97,6 @@ pub struct CodeObject<C: Constant = ConstantData> {
 }
 
 bitflags! {
-    #[derive(Serialize, Deserialize)]
     pub struct CodeFlags: u16 {
         const NEW_LOCALS = 0x01;
         const IS_GENERATOR = 0x02;
@@ -114,7 +121,7 @@ impl CodeFlags {
 }
 
 /// an opcode argument that may be extended by a prior ExtendedArg
-#[derive(Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Copy, Clone, PartialEq, Eq)]
 #[repr(transparent)]
 pub struct OpArgByte(pub u8);
 impl OpArgByte {
@@ -211,23 +218,7 @@ impl OpArgType for bool {
     }
 }
 
-macro_rules! enum_oparg {
-    ($t:ident) => {
-        impl OpArgType for $t {
-            #[inline(always)]
-            fn from_oparg(x: u32) -> Option<Self> {
-                $t::try_from_primitive(x as _).ok()
-            }
-            #[inline(always)]
-            fn to_oparg(self) -> u32 {
-                u8::from(self).into()
-            }
-        }
-    };
-}
-
-#[derive(Copy, Clone, Serialize, Deserialize)]
-#[serde(bound = "")]
+#[derive(Copy, Clone)]
 pub struct Arg<T: OpArgType>(PhantomData<T>);
 
 impl<T: OpArgType> Arg<T> {
@@ -302,7 +293,7 @@ impl fmt::Display for Label {
 }
 
 /// Transforms a value prior to formatting it.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
 #[repr(u8)]
 pub enum ConversionFlag {
     /// No conversion
@@ -314,32 +305,57 @@ pub enum ConversionFlag {
     /// Converts by calling `repr(<value>)`.
     Repr = b'r',
 }
-enum_oparg!(ConversionFlag);
+
+impl OpArgType for ConversionFlag {
+    fn to_oparg(self) -> u32 {
+        self as u32
+    }
+    fn from_oparg(x: u32) -> Option<Self> {
+        Some(match u8::try_from(x).ok()? {
+            0 => Self::None,
+            b's' => Self::Str,
+            b'a' => Self::Ascii,
+            b'r' => Self::Repr,
+            _ => return None,
+        })
+    }
+}
 
 impl TryFrom<usize> for ConversionFlag {
     type Error = usize;
     fn try_from(b: usize) -> Result<Self, Self::Error> {
-        u8::try_from(b)
-            .ok()
-            .and_then(|b| Self::try_from(b).ok())
-            .ok_or(b)
+        u32::try_from(b).ok().and_then(Self::from_oparg).ok_or(b)
     }
 }
 
 /// The kind of Raise that occurred.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
 #[repr(u8)]
 pub enum RaiseKind {
     Reraise,
     Raise,
     RaiseCause,
 }
-enum_oparg!(RaiseKind);
+
+impl OpArgType for RaiseKind {
+    fn to_oparg(self) -> u32 {
+        self as u32
+    }
+    fn from_oparg(x: u32) -> Option<Self> {
+        Some(match x {
+            0 => Self::Reraise,
+            1 => Self::Raise,
+            2 => Self::RaiseCause,
+            _ => return None,
+        })
+    }
+}
 
 pub type NameIdx = u32;
 
 /// A Single bytecode instruction.
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+#[repr(u8)]
 pub enum Instruction {
     /// Importing by name
     ImportName {
@@ -562,7 +578,29 @@ pub enum Instruction {
 }
 const _: () = assert!(mem::size_of::<Instruction>() == 1);
 
-#[derive(Copy, Clone, Serialize, Deserialize)]
+impl From<Instruction> for u8 {
+    #[inline]
+    fn from(ins: Instruction) -> u8 {
+        // SAFETY: there's no padding bits
+        unsafe { std::mem::transmute::<Instruction, u8>(ins) }
+    }
+}
+
+impl TryFrom<u8> for Instruction {
+    type Error = crate::marshal::MarshalError;
+
+    #[inline]
+    fn try_from(value: u8) -> Result<Self, crate::marshal::MarshalError> {
+        if value <= u8::from(Instruction::ExtendedArg) {
+            Ok(unsafe { std::mem::transmute::<u8, Instruction>(value) })
+        } else {
+            Err(crate::marshal::MarshalError::InvalidBytecode)
+        }
+    }
+}
+
+#[derive(Copy, Clone)]
+#[repr(C)]
 pub struct CodeUnit {
     pub op: Instruction,
     pub arg: OpArgByte,
@@ -579,7 +617,6 @@ impl CodeUnit {
 use self::Instruction::*;
 
 bitflags! {
-    #[derive(Serialize, Deserialize)]
     pub struct MakeFunctionFlags: u8 {
         const CLOSURE = 0x01;
         const ANNOTATIONS = 0x02;
@@ -607,7 +644,7 @@ impl OpArgType for MakeFunctionFlags {
 /// let b = ConstantData::Boolean {value: false};
 /// assert_ne!(a, b);
 /// ```
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone)]
 pub enum ConstantData {
     Tuple { elements: Vec<ConstantData> },
     Integer { value: BigInt },
@@ -677,36 +714,40 @@ pub enum BorrowedConstant<'a, C: Constant> {
     Str { value: &'a str },
     Bytes { value: &'a [u8] },
     Code { code: &'a CodeObject<C> },
-    Tuple { elements: BorrowedTupleIter<'a, C> },
+    Tuple { elements: &'a [C] },
     None,
     Ellipsis,
 }
 
-type BorrowedTupleIter<'a, C> = Box<dyn Iterator<Item = BorrowedConstant<'a, C>> + 'a>;
+impl<C: Constant> Copy for BorrowedConstant<'_, C> {}
+impl<C: Constant> Clone for BorrowedConstant<'_, C> {
+    fn clone(&self) -> Self {
+        *self
+    }
+}
 
 impl<C: Constant> BorrowedConstant<'_, C> {
-    // takes `self` because we need to consume the iterator
-    pub fn fmt_display(self, f: &mut fmt::Formatter) -> fmt::Result {
+    pub fn fmt_display(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             BorrowedConstant::Integer { value } => write!(f, "{value}"),
             BorrowedConstant::Float { value } => write!(f, "{value}"),
             BorrowedConstant::Complex { value } => write!(f, "{value}"),
             BorrowedConstant::Boolean { value } => {
-                write!(f, "{}", if value { "True" } else { "False" })
+                write!(f, "{}", if *value { "True" } else { "False" })
             }
             BorrowedConstant::Str { value } => write!(f, "{value:?}"),
-            BorrowedConstant::Bytes { value } => write!(f, "b{:?}", value.as_bstr()),
+            BorrowedConstant::Bytes { value } => write!(f, "b\"{}\"", value.escape_ascii()),
             BorrowedConstant::Code { code } => write!(f, "{code:?}"),
             BorrowedConstant::Tuple { elements } => {
                 write!(f, "(")?;
                 let mut first = true;
-                for c in elements {
+                for c in *elements {
                     if first {
                         first = false
                     } else {
                         write!(f, ", ")?;
                     }
-                    c.fmt_display(f)?;
+                    c.borrow_constant().fmt_display(f)?;
                 }
                 write!(f, ")")
             }
@@ -733,7 +774,10 @@ impl<C: Constant> BorrowedConstant<'_, C> {
                 code: Box::new(code.map_clone_bag(&BasicBag)),
             },
             BorrowedConstant::Tuple { elements } => Tuple {
-                elements: elements.map(BorrowedConstant::to_owned).collect(),
+                elements: elements
+                    .iter()
+                    .map(|c| c.borrow_constant().to_owned())
+                    .collect(),
             },
             BorrowedConstant::None => None,
             BorrowedConstant::Ellipsis => Ellipsis,
@@ -742,7 +786,7 @@ impl<C: Constant> BorrowedConstant<'_, C> {
 }
 
 /// The possible comparison operators
-#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
 #[repr(u8)]
 pub enum ComparisonOperator {
     // be intentional with bits so that we can do eval_ord with just a bitwise and
@@ -754,9 +798,25 @@ pub enum ComparisonOperator {
     LessOrEqual = 0b101,
     GreaterOrEqual = 0b110,
 }
-enum_oparg!(ComparisonOperator);
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
+impl OpArgType for ComparisonOperator {
+    fn to_oparg(self) -> u32 {
+        self as u32
+    }
+    fn from_oparg(x: u32) -> Option<Self> {
+        Some(match x {
+            0b001 => Self::Less,
+            0b010 => Self::Greater,
+            0b011 => Self::NotEqual,
+            0b100 => Self::Equal,
+            0b101 => Self::LessOrEqual,
+            0b110 => Self::GreaterOrEqual,
+            _ => return None,
+        })
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
 #[repr(u8)]
 pub enum TestOperator {
     In,
@@ -766,7 +826,22 @@ pub enum TestOperator {
     /// two exceptions that match?
     ExceptionMatch,
 }
-enum_oparg!(TestOperator);
+
+impl OpArgType for TestOperator {
+    fn to_oparg(self) -> u32 {
+        self as u32
+    }
+    fn from_oparg(x: u32) -> Option<Self> {
+        Some(match x {
+            0 => Self::In,
+            1 => Self::NotIn,
+            2 => Self::Is,
+            3 => Self::IsNot,
+            4 => Self::ExceptionMatch,
+            _ => return None,
+        })
+    }
+}
 
 /// The possible Binary operators
 /// # Examples
@@ -776,7 +851,7 @@ enum_oparg!(TestOperator);
 /// use rustpython_compiler_core::BinaryOperator::Add;
 /// let op = BinaryOperation {op: Add};
 /// ```
-#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
 #[repr(u8)]
 pub enum BinaryOperator {
     Power,
@@ -793,10 +868,33 @@ pub enum BinaryOperator {
     Xor,
     Or,
 }
-enum_oparg!(BinaryOperator);
+
+impl OpArgType for BinaryOperator {
+    fn to_oparg(self) -> u32 {
+        self as u32
+    }
+    fn from_oparg(x: u32) -> Option<Self> {
+        Some(match x {
+            0 => Self::Power,
+            1 => Self::Multiply,
+            2 => Self::MatrixMultiply,
+            3 => Self::Divide,
+            4 => Self::FloorDivide,
+            5 => Self::Modulo,
+            6 => Self::Add,
+            7 => Self::Subtract,
+            8 => Self::Lshift,
+            9 => Self::Rshift,
+            10 => Self::And,
+            11 => Self::Xor,
+            12 => Self::Or,
+            _ => return None,
+        })
+    }
+}
 
 /// The possible unary operators
-#[derive(Debug, Copy, Clone, PartialEq, Eq, TryFromPrimitive, IntoPrimitive)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
 #[repr(u8)]
 pub enum UnaryOperator {
     Not,
@@ -804,7 +902,21 @@ pub enum UnaryOperator {
     Minus,
     Plus,
 }
-enum_oparg!(UnaryOperator);
+
+impl OpArgType for UnaryOperator {
+    fn to_oparg(self) -> u32 {
+        self as u32
+    }
+    fn from_oparg(x: u32) -> Option<Self> {
+        Some(match x {
+            0 => Self::Not,
+            1 => Self::Invert,
+            2 => Self::Minus,
+            3 => Self::Plus,
+            _ => return None,
+        })
+    }
+}
 
 #[derive(Copy, Clone)]
 pub struct UnpackExArgs {
@@ -867,10 +979,10 @@ impl<C: Constant> CodeObject<C> {
     /// Get all arguments of the code object
     /// like inspect.getargs
     pub fn arg_names(&self) -> Arguments<C::Name> {
-        let nargs = self.arg_count;
-        let nkwargs = self.kwonlyarg_count;
+        let nargs = self.arg_count as usize;
+        let nkwargs = self.kwonlyarg_count as usize;
         let mut varargspos = nargs + nkwargs;
-        let posonlyargs = &self.varnames[..self.posonlyarg_count];
+        let posonlyargs = &self.varnames[..self.posonlyarg_count as usize];
         let args = &self.varnames[..nargs];
         let kwonlyargs = &self.varnames[nargs..varargspos];
 
@@ -1033,49 +1145,23 @@ impl<C: Constant> CodeObject<C> {
     }
 }
 
-/// Error that occurs during code deserialization
-#[derive(Debug)]
-#[non_exhaustive]
-pub enum CodeDeserializeError {
-    /// Unexpected End Of File
-    Eof,
-    /// Invalid Bytecode
-    Other,
-}
-
-impl fmt::Display for CodeDeserializeError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            Self::Eof => f.write_str("unexpected end of data"),
-            Self::Other => f.write_str("invalid bytecode"),
-        }
-    }
-}
-
-impl std::error::Error for CodeDeserializeError {}
-
 impl CodeObject<ConstantData> {
     /// Load a code object from bytes
-    pub fn from_bytes(data: &[u8]) -> Result<Self, CodeDeserializeError> {
+    pub fn from_bytes(data: &[u8]) -> Result<Self, MarshalError> {
         use lz4_flex::block::DecompressError;
         let raw_bincode = lz4_flex::decompress_size_prepended(data).map_err(|e| match e {
             DecompressError::OutputTooSmall { .. } | DecompressError::ExpectedAnotherByte => {
-                CodeDeserializeError::Eof
+                MarshalError::Eof
             }
-            _ => CodeDeserializeError::Other,
+            _ => MarshalError::InvalidBytecode,
         })?;
-        let data = bincode::deserialize(&raw_bincode).map_err(|e| match *e {
-            bincode::ErrorKind::Io(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
-                CodeDeserializeError::Eof
-            }
-            _ => CodeDeserializeError::Other,
-        })?;
-        Ok(data)
+        marshal::deserialize_code(&mut &raw_bincode[..], BasicBag)
     }
 
     /// Serialize this bytecode to bytes.
     pub fn to_bytes(&self) -> Vec<u8> {
-        let data = bincode::serialize(&self).expect("CodeObject is not serializable");
+        let mut data = Vec::new();
+        marshal::serialize_code(&mut data, self);
         lz4_flex::compress_prepend_size(&data)
     }
 }
@@ -1466,7 +1552,7 @@ impl<C: Constant> fmt::Debug for CodeObject<C> {
 }
 
 /// A frozen module. Holds a code object and whether it is part of a package
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Debug)]
 pub struct FrozenModule {
     pub code: CodeObject<ConstantData>,
     pub package: bool,
@@ -1474,34 +1560,28 @@ pub struct FrozenModule {
 
 pub mod frozen_lib {
     use super::*;
-    use bincode::{options, Options};
-    use std::io;
+    use marshal::{Read, Write};
 
     /// Decode a library to a iterable of frozen modules
     pub fn decode_lib(bytes: &[u8]) -> FrozenModulesIter {
         let data = lz4_flex::decompress_size_prepended(bytes).unwrap();
-        let r = VecReader { data, pos: 0 };
-        let mut de = bincode::Deserializer::with_bincode_read(r, options());
-        let len = u64::deserialize(&mut de).unwrap().try_into().unwrap();
-        FrozenModulesIter { len, de }
+        let mut data = marshal::Cursor { data, position: 0 };
+        let remaining = data.read_u32().unwrap();
+        FrozenModulesIter { remaining, data }
     }
 
     pub struct FrozenModulesIter {
-        len: usize,
-        // ideally this could be a SeqAccess, but I think that would require existential types
-        de: bincode::Deserializer<VecReader, bincode::DefaultOptions>,
+        remaining: u32,
+        data: marshal::Cursor<Vec<u8>>,
     }
 
     impl Iterator for FrozenModulesIter {
         type Item = (String, FrozenModule);
 
         fn next(&mut self) -> Option<Self::Item> {
-            // manually mimic bincode's seq encoding, which is <len:u64> <element*len>
-            // This probably won't change (bincode doesn't require padding or anything), but
-            // it's not guaranteed by semver as far as I can tell
-            if self.len > 0 {
-                let entry = Deserialize::deserialize(&mut self.de).unwrap();
-                self.len -= 1;
+            if self.remaining > 0 {
+                let entry = read_entry(&mut self.data).unwrap();
+                self.remaining -= 1;
                 Some(entry)
             } else {
                 None
@@ -1509,12 +1589,19 @@ pub mod frozen_lib {
         }
 
         fn size_hint(&self) -> (usize, Option<usize>) {
-            (self.len, Some(self.len))
+            (self.remaining as usize, Some(self.remaining as usize))
         }
     }
-
     impl ExactSizeIterator for FrozenModulesIter {}
 
+    fn read_entry(rdr: &mut impl Read) -> Result<(String, FrozenModule), marshal::MarshalError> {
+        let len = rdr.read_u32()?;
+        let name = rdr.read_str(len)?.to_owned();
+        let code = marshal::deserialize_code(rdr, BasicBag)?;
+        let package = rdr.read_u8()? != 0;
+        Ok((name, FrozenModule { code, package }))
+    }
+
     /// Encode the given iterator of frozen modules into a compressed vector of bytes
     pub fn encode_lib<'a, I>(lib: I) -> Vec<u8>
     where
@@ -1522,82 +1609,25 @@ pub mod frozen_lib {
         I::IntoIter: ExactSizeIterator + Clone,
     {
         let iter = lib.into_iter();
-        let data = options().serialize(&SerializeLib { iter }).unwrap();
+        let mut data = Vec::new();
+        write_lib(&mut data, iter);
         lz4_flex::compress_prepend_size(&data)
     }
 
-    struct SerializeLib<I> {
-        iter: I,
-    }
-
-    impl<'a, I> Serialize for SerializeLib<I>
-    where
-        I: ExactSizeIterator<Item = (&'a str, &'a FrozenModule)> + Clone,
-    {
-        fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-        where
-            S: serde::Serializer,
-        {
-            serializer.collect_seq(self.iter.clone())
-        }
-    }
-
-    /// Owned version of bincode::de::read::SliceReader<'a>
-    struct VecReader {
-        data: Vec<u8>,
-        pos: usize,
-    }
-
-    impl io::Read for VecReader {
-        fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
-            let mut subslice = &self.data[self.pos..];
-            let n = io::Read::read(&mut subslice, buf)?;
-            self.pos += n;
-            Ok(n)
-        }
-        fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> {
-            self.get_byte_slice(buf.len())
-                .map(|data| buf.copy_from_slice(data))
-        }
-    }
-
-    impl VecReader {
-        #[inline(always)]
-        fn get_byte_slice(&mut self, length: usize) -> io::Result<&[u8]> {
-            let subslice = &self.data[self.pos..];
-            match subslice.get(..length) {
-                Some(ret) => {
-                    self.pos += length;
-                    Ok(ret)
-                }
-                None => Err(io::ErrorKind::UnexpectedEof.into()),
-            }
+    fn write_lib<'a>(
+        buf: &mut impl Write,
+        lib: impl ExactSizeIterator<Item = (&'a str, &'a FrozenModule)>,
+    ) {
+        marshal::write_len(buf, lib.len());
+        for (name, module) in lib {
+            write_entry(buf, name, module);
         }
     }
 
-    impl<'storage> bincode::BincodeRead<'storage> for VecReader {
-        fn forward_read_str<V>(&mut self, length: usize, visitor: V) -> bincode::Result<V::Value>
-        where
-            V: serde::de::Visitor<'storage>,
-        {
-            let bytes = self.get_byte_slice(length)?;
-            match ::std::str::from_utf8(bytes) {
-                Ok(s) => visitor.visit_str(s),
-                Err(e) => Err(bincode::ErrorKind::InvalidUtf8Encoding(e).into()),
-            }
-        }
-
-        fn get_byte_buffer(&mut self, length: usize) -> bincode::Result<Vec<u8>> {
-            self.get_byte_slice(length)
-                .map(|x| x.to_vec())
-                .map_err(Into::into)
-        }
-
-        fn forward_read_bytes<V>(&mut self, length: usize, visitor: V) -> bincode::Result<V::Value>
-        where
-            V: serde::de::Visitor<'storage>,
-        {
-            visitor.visit_bytes(self.get_byte_slice(length)?)
-        }
+    fn write_entry(buf: &mut impl Write, name: &str, module: &FrozenModule) {
+        marshal::write_len(buf, name.len());
+        buf.write_slice(name.as_bytes());
+        marshal::serialize_code(buf, &module.code);
+        buf.write_u8(module.package as u8);
     }
 }
diff --git a/core/src/error.rs b/core/src/error.rs
index 9619e1d8..813ca983 100644
--- a/core/src/error.rs
+++ b/core/src/error.rs
@@ -1,4 +1,5 @@
 use crate::Location;
+use std::error::Error as StdError;
 use std::fmt::Display;
 
 #[derive(Debug, PartialEq, Eq)]
@@ -16,7 +17,14 @@ impl<T> std::ops::Deref for BaseError<T> {
     }
 }
 
-impl<T> std::error::Error for BaseError<T> where T: std::fmt::Display + std::fmt::Debug {}
+impl<T> StdError for BaseError<T>
+where
+    T: StdError + 'static,
+{
+    fn source(&self) -> Option<&(dyn StdError + 'static)> {
+        Some(&self.error)
+    }
+}
 
 impl<T> Display for BaseError<T>
 where
@@ -51,12 +59,18 @@ impl<T> BaseError<T> {
     }
 }
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug)]
 pub struct CompileError<T> {
     pub body: BaseError<T>,
     pub statement: Option<String>,
 }
 
+impl<T: StdError + 'static> StdError for CompileError<T> {
+    fn source(&self) -> Option<&(dyn StdError + 'static)> {
+        self.body.source()
+    }
+}
+
 impl<T> std::ops::Deref for CompileError<T> {
     type Target = BaseError<T>;
     fn deref(&self) -> &Self::Target {
diff --git a/core/src/lib.rs b/core/src/lib.rs
index e1f0a5b3..c431095f 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -4,6 +4,7 @@
 mod bytecode;
 mod error;
 mod location;
+pub mod marshal;
 mod mode;
 
 pub use bytecode::*;
diff --git a/core/src/location.rs b/core/src/location.rs
index d9cd59be..2a2d029d 100644
--- a/core/src/location.rs
+++ b/core/src/location.rs
@@ -1,7 +1,5 @@
-use serde::{Deserialize, Serialize};
-
 /// Sourcecode location.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub struct Location {
     pub(super) row: u32,
     pub(super) column: u32,
diff --git a/core/src/marshal.rs b/core/src/marshal.rs
new file mode 100644
index 00000000..69d88793
--- /dev/null
+++ b/core/src/marshal.rs
@@ -0,0 +1,615 @@
+use core::fmt;
+use std::convert::Infallible;
+
+use num_bigint::{BigInt, Sign};
+use num_complex::Complex64;
+
+use crate::{bytecode::*, Location};
+
+pub const FORMAT_VERSION: u32 = 4;
+
+#[derive(Debug)]
+pub enum MarshalError {
+    /// Unexpected End Of File
+    Eof,
+    /// Invalid Bytecode
+    InvalidBytecode,
+    /// Invalid utf8 in string
+    InvalidUtf8,
+    /// Bad type marker
+    BadType,
+}
+
+impl fmt::Display for MarshalError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Self::Eof => f.write_str("unexpected end of data"),
+            Self::InvalidBytecode => f.write_str("invalid bytecode"),
+            Self::InvalidUtf8 => f.write_str("invalid utf8"),
+            Self::BadType => f.write_str("bad type marker"),
+        }
+    }
+}
+
+impl From<std::str::Utf8Error> for MarshalError {
+    fn from(_: std::str::Utf8Error) -> Self {
+        Self::InvalidUtf8
+    }
+}
+
+impl std::error::Error for MarshalError {}
+
+type Result<T, E = MarshalError> = std::result::Result<T, E>;
+
+#[repr(u8)]
+enum Type {
+    // Null = b'0',
+    None = b'N',
+    False = b'F',
+    True = b'T',
+    StopIter = b'S',
+    Ellipsis = b'.',
+    Int = b'i',
+    Float = b'g',
+    Complex = b'y',
+    // Long = b'l',  // i32
+    Bytes = b's', // = TYPE_STRING
+    // Interned = b't',
+    // Ref = b'r',
+    Tuple = b'(',
+    List = b'[',
+    Dict = b'{',
+    Code = b'c',
+    Unicode = b'u',
+    // Unknown = b'?',
+    Set = b'<',
+    FrozenSet = b'>',
+    Ascii = b'a',
+    // AsciiInterned = b'A',
+    // SmallTuple = b')',
+    // ShortAscii = b'z',
+    // ShortAsciiInterned = b'Z',
+}
+// const FLAG_REF: u8 = b'\x80';
+
+impl TryFrom<u8> for Type {
+    type Error = MarshalError;
+    fn try_from(value: u8) -> Result<Self> {
+        use Type::*;
+        Ok(match value {
+            // b'0' => Null,
+            b'N' => None,
+            b'F' => False,
+            b'T' => True,
+            b'S' => StopIter,
+            b'.' => Ellipsis,
+            b'i' => Int,
+            b'g' => Float,
+            b'y' => Complex,
+            // b'l' => Long,
+            b's' => Bytes,
+            // b't' => Interned,
+            // b'r' => Ref,
+            b'(' => Tuple,
+            b'[' => List,
+            b'{' => Dict,
+            b'c' => Code,
+            b'u' => Unicode,
+            // b'?' => Unknown,
+            b'<' => Set,
+            b'>' => FrozenSet,
+            b'a' => Ascii,
+            // b'A' => AsciiInterned,
+            // b')' => SmallTuple,
+            // b'z' => ShortAscii,
+            // b'Z' => ShortAsciiInterned,
+            _ => return Err(MarshalError::BadType),
+        })
+    }
+}
+
+pub trait Read {
+    fn read_slice(&mut self, n: u32) -> Result<&[u8]>;
+    fn read_array<const N: usize>(&mut self) -> Result<&[u8; N]> {
+        self.read_slice(N as u32).map(|s| s.try_into().unwrap())
+    }
+    fn read_str(&mut self, len: u32) -> Result<&str> {
+        Ok(std::str::from_utf8(self.read_slice(len)?)?)
+    }
+    fn read_u8(&mut self) -> Result<u8> {
+        Ok(u8::from_le_bytes(*self.read_array()?))
+    }
+    fn read_u16(&mut self) -> Result<u16> {
+        Ok(u16::from_le_bytes(*self.read_array()?))
+    }
+    fn read_u32(&mut self) -> Result<u32> {
+        Ok(u32::from_le_bytes(*self.read_array()?))
+    }
+    fn read_u64(&mut self) -> Result<u64> {
+        Ok(u64::from_le_bytes(*self.read_array()?))
+    }
+}
+
+impl Read for &[u8] {
+    fn read_slice(&mut self, n: u32) -> Result<&[u8]> {
+        let data = self.get(..n as usize).ok_or(MarshalError::Eof)?;
+        *self = &self[n as usize..];
+        Ok(data)
+    }
+}
+
+pub struct Cursor<B> {
+    pub data: B,
+    pub position: usize,
+}
+
+impl<B: AsRef<[u8]>> Read for Cursor<B> {
+    fn read_slice(&mut self, n: u32) -> Result<&[u8]> {
+        let data = &self.data.as_ref()[self.position..];
+        let slice = data.get(..n as usize).ok_or(MarshalError::Eof)?;
+        self.position += n as usize;
+        Ok(slice)
+    }
+}
+
+pub fn deserialize_code<R: Read, Bag: ConstantBag>(
+    rdr: &mut R,
+    bag: Bag,
+) -> Result<CodeObject<Bag::Constant>> {
+    let len = rdr.read_u32()?;
+    let instructions = rdr.read_slice(len * 2)?;
+    let instructions = instructions
+        .chunks_exact(2)
+        .map(|cu| {
+            let op = Instruction::try_from(cu[0])?;
+            let arg = OpArgByte(cu[1]);
+            Ok(CodeUnit { op, arg })
+        })
+        .collect::<Result<Box<[CodeUnit]>>>()?;
+
+    let len = rdr.read_u32()?;
+    let locations = (0..len)
+        .map(|_| {
+            Ok(Location {
+                row: rdr.read_u32()?,
+                column: rdr.read_u32()?,
+            })
+        })
+        .collect::<Result<Box<[Location]>>>()?;
+
+    let flags = CodeFlags::from_bits_truncate(rdr.read_u16()?);
+
+    let posonlyarg_count = rdr.read_u32()?;
+    let arg_count = rdr.read_u32()?;
+    let kwonlyarg_count = rdr.read_u32()?;
+
+    let len = rdr.read_u32()?;
+    let source_path = bag.make_name(rdr.read_str(len)?);
+
+    let first_line_number = rdr.read_u32()?;
+    let max_stackdepth = rdr.read_u32()?;
+
+    let len = rdr.read_u32()?;
+    let obj_name = bag.make_name(rdr.read_str(len)?);
+
+    let len = rdr.read_u32()?;
+    let cell2arg = (len != 0)
+        .then(|| {
+            (0..len)
+                .map(|_| Ok(rdr.read_u32()? as i32))
+                .collect::<Result<Box<[i32]>>>()
+        })
+        .transpose()?;
+
+    let len = rdr.read_u32()?;
+    let constants = (0..len)
+        .map(|_| deserialize_value(rdr, bag))
+        .collect::<Result<Box<[_]>>>()?;
+
+    let mut read_names = || {
+        let len = rdr.read_u32()?;
+        (0..len)
+            .map(|_| {
+                let len = rdr.read_u32()?;
+                Ok(bag.make_name(rdr.read_str(len)?))
+            })
+            .collect::<Result<Box<[_]>>>()
+    };
+
+    let names = read_names()?;
+    let varnames = read_names()?;
+    let cellvars = read_names()?;
+    let freevars = read_names()?;
+
+    Ok(CodeObject {
+        instructions,
+        locations,
+        flags,
+        posonlyarg_count,
+        arg_count,
+        kwonlyarg_count,
+        source_path,
+        first_line_number,
+        max_stackdepth,
+        obj_name,
+        cell2arg,
+        constants,
+        names,
+        varnames,
+        cellvars,
+        freevars,
+    })
+}
+
+pub trait MarshalBag: Copy {
+    type Value;
+    fn make_bool(&self, value: bool) -> Self::Value;
+    fn make_none(&self) -> Self::Value;
+    fn make_ellipsis(&self) -> Self::Value;
+    fn make_float(&self, value: f64) -> Self::Value;
+    fn make_complex(&self, value: Complex64) -> Self::Value;
+    fn make_str(&self, value: &str) -> Self::Value;
+    fn make_bytes(&self, value: &[u8]) -> Self::Value;
+    fn make_int(&self, value: BigInt) -> Self::Value;
+    fn make_tuple(&self, elements: impl Iterator<Item = Self::Value>) -> Self::Value;
+    fn make_code(
+        &self,
+        code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
+    ) -> Self::Value;
+    fn make_stop_iter(&self) -> Result<Self::Value>;
+    fn make_list(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>;
+    fn make_set(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>;
+    fn make_frozenset(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>;
+    fn make_dict(
+        &self,
+        it: impl Iterator<Item = (Self::Value, Self::Value)>,
+    ) -> Result<Self::Value>;
+    type ConstantBag: ConstantBag;
+    fn constant_bag(self) -> Self::ConstantBag;
+}
+
+impl<Bag: ConstantBag> MarshalBag for Bag {
+    type Value = Bag::Constant;
+    fn make_bool(&self, value: bool) -> Self::Value {
+        self.make_constant::<Bag::Constant>(BorrowedConstant::Boolean { value })
+    }
+    fn make_none(&self) -> Self::Value {
+        self.make_constant::<Bag::Constant>(BorrowedConstant::None)
+    }
+    fn make_ellipsis(&self) -> Self::Value {
+        self.make_constant::<Bag::Constant>(BorrowedConstant::Ellipsis)
+    }
+    fn make_float(&self, value: f64) -> Self::Value {
+        self.make_constant::<Bag::Constant>(BorrowedConstant::Float { value })
+    }
+    fn make_complex(&self, value: Complex64) -> Self::Value {
+        self.make_constant::<Bag::Constant>(BorrowedConstant::Complex { value })
+    }
+    fn make_str(&self, value: &str) -> Self::Value {
+        self.make_constant::<Bag::Constant>(BorrowedConstant::Str { value })
+    }
+    fn make_bytes(&self, value: &[u8]) -> Self::Value {
+        self.make_constant::<Bag::Constant>(BorrowedConstant::Bytes { value })
+    }
+    fn make_int(&self, value: BigInt) -> Self::Value {
+        self.make_int(value)
+    }
+    fn make_tuple(&self, elements: impl Iterator<Item = Self::Value>) -> Self::Value {
+        self.make_tuple(elements)
+    }
+    fn make_code(
+        &self,
+        code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
+    ) -> Self::Value {
+        self.make_code(code)
+    }
+    fn make_stop_iter(&self) -> Result<Self::Value> {
+        Err(MarshalError::BadType)
+    }
+    fn make_list(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> {
+        Err(MarshalError::BadType)
+    }
+    fn make_set(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> {
+        Err(MarshalError::BadType)
+    }
+    fn make_frozenset(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> {
+        Err(MarshalError::BadType)
+    }
+    fn make_dict(
+        &self,
+        _: impl Iterator<Item = (Self::Value, Self::Value)>,
+    ) -> Result<Self::Value> {
+        Err(MarshalError::BadType)
+    }
+    type ConstantBag = Self;
+    fn constant_bag(self) -> Self::ConstantBag {
+        self
+    }
+}
+
+pub fn deserialize_value<R: Read, Bag: MarshalBag>(rdr: &mut R, bag: Bag) -> Result<Bag::Value> {
+    let typ = Type::try_from(rdr.read_u8()?)?;
+    let value = match typ {
+        Type::True => bag.make_bool(true),
+        Type::False => bag.make_bool(false),
+        Type::None => bag.make_none(),
+        Type::StopIter => bag.make_stop_iter()?,
+        Type::Ellipsis => bag.make_ellipsis(),
+        Type::Int => {
+            let len = rdr.read_u32()? as i32;
+            let sign = if len < 0 { Sign::Minus } else { Sign::Plus };
+            let bytes = rdr.read_slice(len.unsigned_abs())?;
+            let int = BigInt::from_bytes_le(sign, bytes);
+            bag.make_int(int)
+        }
+        Type::Float => {
+            let value = f64::from_bits(rdr.read_u64()?);
+            bag.make_float(value)
+        }
+        Type::Complex => {
+            let re = f64::from_bits(rdr.read_u64()?);
+            let im = f64::from_bits(rdr.read_u64()?);
+            let value = Complex64 { re, im };
+            bag.make_complex(value)
+        }
+        Type::Ascii | Type::Unicode => {
+            let len = rdr.read_u32()?;
+            let value = rdr.read_str(len)?;
+            bag.make_str(value)
+        }
+        Type::Tuple => {
+            let len = rdr.read_u32()?;
+            let it = (0..len).map(|_| deserialize_value(rdr, bag));
+            itertools::process_results(it, |it| bag.make_tuple(it))?
+        }
+        Type::List => {
+            let len = rdr.read_u32()?;
+            let it = (0..len).map(|_| deserialize_value(rdr, bag));
+            itertools::process_results(it, |it| bag.make_list(it))??
+        }
+        Type::Set => {
+            let len = rdr.read_u32()?;
+            let it = (0..len).map(|_| deserialize_value(rdr, bag));
+            itertools::process_results(it, |it| bag.make_set(it))??
+        }
+        Type::FrozenSet => {
+            let len = rdr.read_u32()?;
+            let it = (0..len).map(|_| deserialize_value(rdr, bag));
+            itertools::process_results(it, |it| bag.make_frozenset(it))??
+        }
+        Type::Dict => {
+            let len = rdr.read_u32()?;
+            let it = (0..len).map(|_| {
+                let k = deserialize_value(rdr, bag)?;
+                let v = deserialize_value(rdr, bag)?;
+                Ok::<_, MarshalError>((k, v))
+            });
+            itertools::process_results(it, |it| bag.make_dict(it))??
+        }
+        Type::Bytes => {
+            // Following CPython, after marshaling, byte arrays are converted into bytes.
+            let len = rdr.read_u32()?;
+            let value = rdr.read_slice(len)?;
+            bag.make_bytes(value)
+        }
+        Type::Code => bag.make_code(deserialize_code(rdr, bag.constant_bag())?),
+    };
+    Ok(value)
+}
+
+pub trait Dumpable: Sized {
+    type Error;
+    type Constant: Constant;
+    fn with_dump<R>(&self, f: impl FnOnce(DumpableValue<'_, Self>) -> R) -> Result<R, Self::Error>;
+}
+
+pub enum DumpableValue<'a, D: Dumpable> {
+    Integer(&'a BigInt),
+    Float(f64),
+    Complex(Complex64),
+    Boolean(bool),
+    Str(&'a str),
+    Bytes(&'a [u8]),
+    Code(&'a CodeObject<D::Constant>),
+    Tuple(&'a [D]),
+    None,
+    Ellipsis,
+    StopIter,
+    List(&'a [D]),
+    Set(&'a [D]),
+    Frozenset(&'a [D]),
+    Dict(&'a [(D, D)]),
+}
+
+impl<'a, C: Constant> From<BorrowedConstant<'a, C>> for DumpableValue<'a, C> {
+    fn from(c: BorrowedConstant<'a, C>) -> Self {
+        match c {
+            BorrowedConstant::Integer { value } => Self::Integer(value),
+            BorrowedConstant::Float { value } => Self::Float(value),
+            BorrowedConstant::Complex { value } => Self::Complex(value),
+            BorrowedConstant::Boolean { value } => Self::Boolean(value),
+            BorrowedConstant::Str { value } => Self::Str(value),
+            BorrowedConstant::Bytes { value } => Self::Bytes(value),
+            BorrowedConstant::Code { code } => Self::Code(code),
+            BorrowedConstant::Tuple { elements } => Self::Tuple(elements),
+            BorrowedConstant::None => Self::None,
+            BorrowedConstant::Ellipsis => Self::Ellipsis,
+        }
+    }
+}
+
+impl<C: Constant> Dumpable for C {
+    type Error = Infallible;
+    type Constant = Self;
+    #[inline(always)]
+    fn with_dump<R>(&self, f: impl FnOnce(DumpableValue<'_, Self>) -> R) -> Result<R, Self::Error> {
+        Ok(f(self.borrow_constant().into()))
+    }
+}
+
+pub trait Write {
+    fn write_slice(&mut self, slice: &[u8]);
+    fn write_u8(&mut self, v: u8) {
+        self.write_slice(&v.to_le_bytes())
+    }
+    fn write_u16(&mut self, v: u16) {
+        self.write_slice(&v.to_le_bytes())
+    }
+    fn write_u32(&mut self, v: u32) {
+        self.write_slice(&v.to_le_bytes())
+    }
+    fn write_u64(&mut self, v: u64) {
+        self.write_slice(&v.to_le_bytes())
+    }
+}
+
+impl Write for Vec<u8> {
+    fn write_slice(&mut self, slice: &[u8]) {
+        self.extend_from_slice(slice)
+    }
+}
+
+pub(crate) fn write_len<W: Write>(buf: &mut W, len: usize) {
+    let Ok(len) = len.try_into() else { panic!("too long to serialize") };
+    buf.write_u32(len);
+}
+
+pub fn serialize_value<W: Write, D: Dumpable>(
+    buf: &mut W,
+    constant: DumpableValue<'_, D>,
+) -> Result<(), D::Error> {
+    match constant {
+        DumpableValue::Integer(int) => {
+            buf.write_u8(Type::Int as u8);
+            let (sign, bytes) = int.to_bytes_le();
+            let len: i32 = bytes.len().try_into().expect("too long to serialize");
+            let len = if sign == Sign::Minus { -len } else { len };
+            buf.write_u32(len as u32);
+            buf.write_slice(&bytes);
+        }
+        DumpableValue::Float(f) => {
+            buf.write_u8(Type::Float as u8);
+            buf.write_u64(f.to_bits());
+        }
+        DumpableValue::Complex(c) => {
+            buf.write_u8(Type::Complex as u8);
+            buf.write_u64(c.re.to_bits());
+            buf.write_u64(c.im.to_bits());
+        }
+        DumpableValue::Boolean(b) => {
+            buf.write_u8(if b { Type::True } else { Type::False } as u8);
+        }
+        DumpableValue::Str(s) => {
+            buf.write_u8(Type::Unicode as u8);
+            write_len(buf, s.len());
+            buf.write_slice(s.as_bytes());
+        }
+        DumpableValue::Bytes(b) => {
+            buf.write_u8(Type::Bytes as u8);
+            write_len(buf, b.len());
+            buf.write_slice(b);
+        }
+        DumpableValue::Code(c) => {
+            buf.write_u8(Type::Code as u8);
+            serialize_code(buf, c);
+        }
+        DumpableValue::Tuple(tup) => {
+            buf.write_u8(Type::Tuple as u8);
+            write_len(buf, tup.len());
+            for val in tup {
+                val.with_dump(|val| serialize_value(buf, val))??
+            }
+        }
+        DumpableValue::None => {
+            buf.write_u8(Type::None as u8);
+        }
+        DumpableValue::Ellipsis => {
+            buf.write_u8(Type::Ellipsis as u8);
+        }
+        DumpableValue::StopIter => {
+            buf.write_u8(Type::StopIter as u8);
+        }
+        DumpableValue::List(l) => {
+            buf.write_u8(Type::List as u8);
+            write_len(buf, l.len());
+            for val in l {
+                val.with_dump(|val| serialize_value(buf, val))??
+            }
+        }
+        DumpableValue::Set(set) => {
+            buf.write_u8(Type::Set as u8);
+            write_len(buf, set.len());
+            for val in set {
+                val.with_dump(|val| serialize_value(buf, val))??
+            }
+        }
+        DumpableValue::Frozenset(set) => {
+            buf.write_u8(Type::FrozenSet as u8);
+            write_len(buf, set.len());
+            for val in set {
+                val.with_dump(|val| serialize_value(buf, val))??
+            }
+        }
+        DumpableValue::Dict(d) => {
+            buf.write_u8(Type::Dict as u8);
+            write_len(buf, d.len());
+            for (k, v) in d {
+                k.with_dump(|val| serialize_value(buf, val))??;
+                v.with_dump(|val| serialize_value(buf, val))??;
+            }
+        }
+    }
+    Ok(())
+}
+
+pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>) {
+    write_len(buf, code.instructions.len());
+    // SAFETY: it's ok to transmute CodeUnit to [u8; 2]
+    let (_, instructions_bytes, _) = unsafe { code.instructions.align_to() };
+    buf.write_slice(instructions_bytes);
+
+    write_len(buf, code.locations.len());
+    for loc in &*code.locations {
+        buf.write_u32(loc.row);
+        buf.write_u32(loc.column);
+    }
+
+    buf.write_u16(code.flags.bits());
+
+    buf.write_u32(code.posonlyarg_count);
+    buf.write_u32(code.arg_count);
+    buf.write_u32(code.kwonlyarg_count);
+
+    write_len(buf, code.source_path.as_ref().len());
+    buf.write_slice(code.source_path.as_ref().as_bytes());
+
+    buf.write_u32(code.first_line_number);
+    buf.write_u32(code.max_stackdepth);
+
+    write_len(buf, code.obj_name.as_ref().len());
+    buf.write_slice(code.obj_name.as_ref().as_bytes());
+
+    let cell2arg = code.cell2arg.as_deref().unwrap_or(&[]);
+    write_len(buf, cell2arg.len());
+    for &i in cell2arg {
+        buf.write_u32(i as u32)
+    }
+
+    write_len(buf, code.constants.len());
+    for constant in &*code.constants {
+        serialize_value(buf, constant.borrow_constant().into()).unwrap_or_else(|x| match x {})
+    }
+
+    let mut write_names = |names: &[C::Name]| {
+        write_len(buf, names.len());
+        for name in names {
+            write_len(buf, name.as_ref().len());
+            buf.write_slice(name.as_ref().as_bytes());
+        }
+    };
+
+    write_names(&code.names);
+    write_names(&code.varnames);
+    write_names(&code.cellvars);
+    write_names(&code.freevars);
+}
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index 40bf75e5..026616d3 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -26,7 +26,6 @@ itertools = { workspace = true }
 log = { workspace = true }
 num-bigint = { workspace = true }
 num-traits = { workspace = true }
-thiserror = { workspace = true }
 unicode_names2 = { workspace = true }
 
 unic-emoji-char = "0.9.0"
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index c2c4a383..04875c44 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -200,7 +200,7 @@ pub fn parse_tokens(
 pub type ParseError = rustpython_compiler_core::BaseError<ParseErrorType>;
 
 /// Represents the different types of errors that can occur during parsing.
-#[derive(Debug, PartialEq, thiserror::Error)]
+#[derive(Debug, PartialEq)]
 pub enum ParseErrorType {
     /// Parser encountered an unexpected end of input
     Eof,
@@ -215,6 +215,8 @@ pub enum ParseErrorType {
     Lexical(LexicalErrorType),
 }
 
+impl std::error::Error for ParseErrorType {}
+
 // Convert `lalrpop_util::ParseError` to our internal type
 fn parse_error_from_lalrpop(
     err: LalrpopError<Location, Tok, LexicalError>,
diff --git a/src/lib.rs b/src/lib.rs
index 804ce10e..27c15872 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -7,12 +7,40 @@ use rustpython_parser::{
 pub use rustpython_codegen::compile::CompileOpts;
 pub use rustpython_compiler_core::{BaseError as CompileErrorBody, CodeObject, Mode};
 
-#[derive(Debug, thiserror::Error)]
+use std::error::Error as StdError;
+use std::fmt;
+
+#[derive(Debug)]
 pub enum CompileErrorType {
-    #[error(transparent)]
-    Codegen(#[from] rustpython_codegen::error::CodegenErrorType),
-    #[error(transparent)]
-    Parse(#[from] parser::ParseErrorType),
+    Codegen(rustpython_codegen::error::CodegenErrorType),
+    Parse(parser::ParseErrorType),
+}
+
+impl StdError for CompileErrorType {
+    fn source(&self) -> Option<&(dyn StdError + 'static)> {
+        match self {
+            CompileErrorType::Codegen(e) => e.source(),
+            CompileErrorType::Parse(e) => e.source(),
+        }
+    }
+}
+impl fmt::Display for CompileErrorType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            CompileErrorType::Codegen(e) => e.fmt(f),
+            CompileErrorType::Parse(e) => e.fmt(f),
+        }
+    }
+}
+impl From<rustpython_codegen::error::CodegenErrorType> for CompileErrorType {
+    fn from(source: rustpython_codegen::error::CodegenErrorType) -> Self {
+        CompileErrorType::Codegen(source)
+    }
+}
+impl From<parser::ParseErrorType> for CompileErrorType {
+    fn from(source: parser::ParseErrorType) -> Self {
+        CompileErrorType::Parse(source)
+    }
 }
 
 pub type CompileError = rustpython_compiler_core::CompileError<CompileErrorType>;

From 461ef3d71d2a2d66f93446594351e076a9964de6 Mon Sep 17 00:00:00 2001
From: Noa <coolreader18@gmail.com>
Date: Wed, 1 Mar 2023 21:05:59 -0600
Subject: [PATCH 443/459] Address review comments

---
 core/src/bytecode.rs | 294 +++++++++++++++++--------------------------
 1 file changed, 113 insertions(+), 181 deletions(-)

diff --git a/core/src/bytecode.rs b/core/src/bytecode.rs
index fa7146d7..3b320318 100644
--- a/core/src/bytecode.rs
+++ b/core/src/bytecode.rs
@@ -218,6 +218,27 @@ impl OpArgType for bool {
     }
 }
 
+macro_rules! oparg_enum {
+    ($(#[$attr:meta])* $vis:vis enum $name:ident { $($(#[$var_attr:meta])* $var:ident = $discr:literal,)* }) => {
+        $(#[$attr])*
+        $vis enum $name {
+            $($(#[$var_attr])* $var = $discr,)*
+        }
+
+        impl OpArgType for $name {
+            fn to_oparg(self) -> u32 {
+                self as u32
+            }
+            fn from_oparg(x: u32) -> Option<Self> {
+                Some(match u8::try_from(x).ok()? {
+                    $($discr => Self::$var,)*
+                    _ => return None,
+                })
+            }
+        }
+    };
+}
+
 #[derive(Copy, Clone)]
 pub struct Arg<T: OpArgType>(PhantomData<T>);
 
@@ -292,34 +313,21 @@ impl fmt::Display for Label {
     }
 }
 
-/// Transforms a value prior to formatting it.
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-#[repr(u8)]
-pub enum ConversionFlag {
-    /// No conversion
-    None = 0, // CPython uses -1 but not pleasure for us
-    /// Converts by calling `str(<value>)`.
-    Str = b's',
-    /// Converts by calling `ascii(<value>)`.
-    Ascii = b'a',
-    /// Converts by calling `repr(<value>)`.
-    Repr = b'r',
-}
-
-impl OpArgType for ConversionFlag {
-    fn to_oparg(self) -> u32 {
-        self as u32
-    }
-    fn from_oparg(x: u32) -> Option<Self> {
-        Some(match u8::try_from(x).ok()? {
-            0 => Self::None,
-            b's' => Self::Str,
-            b'a' => Self::Ascii,
-            b'r' => Self::Repr,
-            _ => return None,
-        })
-    }
-}
+oparg_enum!(
+    /// Transforms a value prior to formatting it.
+    #[derive(Copy, Clone, Debug, PartialEq, Eq)]
+    #[repr(u8)]
+    pub enum ConversionFlag {
+        /// No conversion
+        None = 0, // CPython uses -1 but not pleasure for us
+        /// Converts by calling `str(<value>)`.
+        Str = b's',
+        /// Converts by calling `ascii(<value>)`.
+        Ascii = b'a',
+        /// Converts by calling `repr(<value>)`.
+        Repr = b'r',
+    }
+);
 
 impl TryFrom<usize> for ConversionFlag {
     type Error = usize;
@@ -328,28 +336,16 @@ impl TryFrom<usize> for ConversionFlag {
     }
 }
 
-/// The kind of Raise that occurred.
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-#[repr(u8)]
-pub enum RaiseKind {
-    Reraise,
-    Raise,
-    RaiseCause,
-}
-
-impl OpArgType for RaiseKind {
-    fn to_oparg(self) -> u32 {
-        self as u32
+oparg_enum!(
+    /// The kind of Raise that occurred.
+    #[derive(Copy, Clone, Debug, PartialEq, Eq)]
+    #[repr(u8)]
+    pub enum RaiseKind {
+        Reraise = 0,
+        Raise = 1,
+        RaiseCause = 2,
     }
-    fn from_oparg(x: u32) -> Option<Self> {
-        Some(match x {
-            0 => Self::Reraise,
-            1 => Self::Raise,
-            2 => Self::RaiseCause,
-            _ => return None,
-        })
-    }
-}
+);
 
 pub type NameIdx = u32;
 
@@ -785,138 +781,74 @@ impl<C: Constant> BorrowedConstant<'_, C> {
     }
 }
 
-/// The possible comparison operators
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-#[repr(u8)]
-pub enum ComparisonOperator {
-    // be intentional with bits so that we can do eval_ord with just a bitwise and
-    // bits: | Equal | Greater | Less |
-    Less = 0b001,
-    Greater = 0b010,
-    NotEqual = 0b011,
-    Equal = 0b100,
-    LessOrEqual = 0b101,
-    GreaterOrEqual = 0b110,
-}
-
-impl OpArgType for ComparisonOperator {
-    fn to_oparg(self) -> u32 {
-        self as u32
-    }
-    fn from_oparg(x: u32) -> Option<Self> {
-        Some(match x {
-            0b001 => Self::Less,
-            0b010 => Self::Greater,
-            0b011 => Self::NotEqual,
-            0b100 => Self::Equal,
-            0b101 => Self::LessOrEqual,
-            0b110 => Self::GreaterOrEqual,
-            _ => return None,
-        })
-    }
-}
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-#[repr(u8)]
-pub enum TestOperator {
-    In,
-    NotIn,
-    Is,
-    IsNot,
-    /// two exceptions that match?
-    ExceptionMatch,
-}
-
-impl OpArgType for TestOperator {
-    fn to_oparg(self) -> u32 {
-        self as u32
-    }
-    fn from_oparg(x: u32) -> Option<Self> {
-        Some(match x {
-            0 => Self::In,
-            1 => Self::NotIn,
-            2 => Self::Is,
-            3 => Self::IsNot,
-            4 => Self::ExceptionMatch,
-            _ => return None,
-        })
-    }
-}
-
-/// The possible Binary operators
-/// # Examples
-///
-/// ```ignore
-/// use rustpython_compiler_core::Instruction::BinaryOperation;
-/// use rustpython_compiler_core::BinaryOperator::Add;
-/// let op = BinaryOperation {op: Add};
-/// ```
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-#[repr(u8)]
-pub enum BinaryOperator {
-    Power,
-    Multiply,
-    MatrixMultiply,
-    Divide,
-    FloorDivide,
-    Modulo,
-    Add,
-    Subtract,
-    Lshift,
-    Rshift,
-    And,
-    Xor,
-    Or,
-}
-
-impl OpArgType for BinaryOperator {
-    fn to_oparg(self) -> u32 {
-        self as u32
-    }
-    fn from_oparg(x: u32) -> Option<Self> {
-        Some(match x {
-            0 => Self::Power,
-            1 => Self::Multiply,
-            2 => Self::MatrixMultiply,
-            3 => Self::Divide,
-            4 => Self::FloorDivide,
-            5 => Self::Modulo,
-            6 => Self::Add,
-            7 => Self::Subtract,
-            8 => Self::Lshift,
-            9 => Self::Rshift,
-            10 => Self::And,
-            11 => Self::Xor,
-            12 => Self::Or,
-            _ => return None,
-        })
-    }
-}
-
-/// The possible unary operators
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-#[repr(u8)]
-pub enum UnaryOperator {
-    Not,
-    Invert,
-    Minus,
-    Plus,
-}
-
-impl OpArgType for UnaryOperator {
-    fn to_oparg(self) -> u32 {
-        self as u32
-    }
-    fn from_oparg(x: u32) -> Option<Self> {
-        Some(match x {
-            0 => Self::Not,
-            1 => Self::Invert,
-            2 => Self::Minus,
-            3 => Self::Plus,
-            _ => return None,
-        })
-    }
-}
+oparg_enum!(
+    /// The possible comparison operators
+    #[derive(Debug, Copy, Clone, PartialEq, Eq)]
+    #[repr(u8)]
+    pub enum ComparisonOperator {
+        // be intentional with bits so that we can do eval_ord with just a bitwise and
+        // bits: | Equal | Greater | Less |
+        Less = 0b001,
+        Greater = 0b010,
+        NotEqual = 0b011,
+        Equal = 0b100,
+        LessOrEqual = 0b101,
+        GreaterOrEqual = 0b110,
+    }
+);
+
+oparg_enum!(
+    #[derive(Debug, Copy, Clone, PartialEq, Eq)]
+    #[repr(u8)]
+    pub enum TestOperator {
+        In = 0,
+        NotIn = 1,
+        Is = 2,
+        IsNot = 3,
+        /// two exceptions that match?
+        ExceptionMatch = 4,
+    }
+);
+
+oparg_enum!(
+    /// The possible Binary operators
+    /// # Examples
+    ///
+    /// ```ignore
+    /// use rustpython_compiler_core::Instruction::BinaryOperation;
+    /// use rustpython_compiler_core::BinaryOperator::Add;
+    /// let op = BinaryOperation {op: Add};
+    /// ```
+    #[derive(Debug, Copy, Clone, PartialEq, Eq)]
+    #[repr(u8)]
+    pub enum BinaryOperator {
+        Power = 0,
+        Multiply = 1,
+        MatrixMultiply = 2,
+        Divide = 3,
+        FloorDivide = 4,
+        Modulo = 5,
+        Add = 6,
+        Subtract = 7,
+        Lshift = 8,
+        Rshift = 9,
+        And = 10,
+        Xor = 11,
+        Or = 12,
+    }
+);
+
+oparg_enum!(
+    /// The possible unary operators
+    #[derive(Debug, Copy, Clone, PartialEq, Eq)]
+    #[repr(u8)]
+    pub enum UnaryOperator {
+        Not = 0,
+        Invert = 1,
+        Minus = 2,
+        Plus = 3,
+    }
+);
 
 #[derive(Copy, Clone)]
 pub struct UnpackExArgs {

From 0cd04ca2de16d7d5d6ba6262430cce31d5ec69c5 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Thu, 2 Mar 2023 13:13:03 +0900
Subject: [PATCH 444/459] usize::to_u32 for compiler internal conversion

---
 codegen/src/compile.rs | 54 ++++++++++++++++++++++++++----------------
 1 file changed, 33 insertions(+), 21 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 2e80de4e..803ac803 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -5,6 +5,8 @@
 //!   <https://github.com/python/cpython/blob/main/Python/compile.c>
 //!   <https://github.com/micropython/micropython/blob/master/py/compile.c>
 
+#![deny(clippy::cast_possible_truncation)]
+
 use crate::{
     error::{CodegenError, CodegenErrorType},
     ir,
@@ -332,7 +334,8 @@ impl Compiler {
         let cache = cache(self.current_codeinfo());
         cache
             .get_index_of(name.as_ref())
-            .unwrap_or_else(|| cache.insert_full(name.into_owned()).0) as u32
+            .unwrap_or_else(|| cache.insert_full(name.into_owned()).0)
+            .to_u32()
     }
 
     fn compile_program(
@@ -900,7 +903,7 @@ impl Compiler {
         let have_defaults = !args.defaults.is_empty();
         if have_defaults {
             // Construct a tuple:
-            let size = args.defaults.len() as u32;
+            let size = args.defaults.len().to_u32();
             for element in &args.defaults {
                 self.compile_expression(element)?;
             }
@@ -921,7 +924,7 @@ impl Compiler {
             emit!(
                 self,
                 Instruction::BuildMap {
-                    size: args.kw_defaults.len() as u32,
+                    size: args.kw_defaults.len().to_u32(),
                 }
             );
         }
@@ -936,11 +939,9 @@ impl Compiler {
 
         self.push_output(
             bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED,
-            args.posonlyargs.len().try_into().unwrap(),
-            (args.posonlyargs.len() + args.args.len())
-                .try_into()
-                .unwrap(),
-            args.kwonlyargs.len().try_into().unwrap(),
+            args.posonlyargs.len().to_u32(),
+            (args.posonlyargs.len() + args.args.len()).to_u32(),
+            args.kwonlyargs.len().to_u32(),
             name.to_owned(),
         );
 
@@ -1269,12 +1270,12 @@ impl Compiler {
             if let SymbolScope::Free = symbol.scope {
                 idx += parent_code.cellvar_cache.len();
             }
-            emit!(self, Instruction::LoadClosure(idx as u32))
+            emit!(self, Instruction::LoadClosure(idx.to_u32()))
         }
         emit!(
             self,
             Instruction::BuildTuple {
-                size: code.freevars.len() as u32,
+                size: code.freevars.len().to_u32(),
             }
         );
         true
@@ -1370,7 +1371,7 @@ impl Compiler {
             .position(|var| *var == "__class__");
 
         if let Some(classcell_idx) = classcell_idx {
-            emit!(self, Instruction::LoadClosure(classcell_idx as u32));
+            emit!(self, Instruction::LoadClosure(classcell_idx.to_u32()));
             emit!(self, Instruction::Duplicate);
             let classcell = self.name("__classcell__");
             emit!(self, Instruction::StoreLocal(classcell));
@@ -1775,7 +1776,7 @@ impl Compiler {
                     emit!(
                         self,
                         Instruction::UnpackSequence {
-                            size: elts.len() as u32,
+                            size: elts.len().to_u32(),
                         }
                     );
                 }
@@ -2175,7 +2176,7 @@ impl Compiler {
                     emit!(
                         self,
                         Instruction::BuildString {
-                            size: values.len() as u32,
+                            size: values.len().to_u32(),
                         }
                     )
                 }
@@ -2244,7 +2245,7 @@ impl Compiler {
                         emit!(
                             compiler,
                             Instruction::ListAppend {
-                                i: generators.len() as u32,
+                                i: generators.len().to_u32(),
                             }
                         );
                         Ok(())
@@ -2263,7 +2264,7 @@ impl Compiler {
                         emit!(
                             compiler,
                             Instruction::SetAdd {
-                                i: generators.len() as u32,
+                                i: generators.len().to_u32(),
                             }
                         );
                         Ok(())
@@ -2289,7 +2290,7 @@ impl Compiler {
                         emit!(
                             compiler,
                             Instruction::MapAdd {
-                                i: generators.len() as u32,
+                                i: generators.len().to_u32(),
                             }
                         );
 
@@ -2420,7 +2421,7 @@ impl Compiler {
         args: &[ast::Expr],
         keywords: &[ast::Keyword],
     ) -> CompileResult<CallType> {
-        let count = (args.len() + keywords.len()) as u32 + additional_positional;
+        let count = (args.len() + keywords.len()).to_u32() + additional_positional;
 
         // Normal arguments:
         let (size, unpack) = self.gather_elements(additional_positional, args)?;
@@ -2521,7 +2522,7 @@ impl Compiler {
             for element in elements {
                 self.compile_expression(element)?;
             }
-            before + elements.len() as u32
+            before + elements.len().to_u32()
         };
 
         Ok((size, has_stars))
@@ -2710,7 +2711,7 @@ impl Compiler {
 
     fn emit_constant(&mut self, constant: ConstantData) {
         let info = self.current_codeinfo();
-        let idx = info.constants.insert_full(constant).0 as u32;
+        let idx = info.constants.insert_full(constant).0.to_u32();
         self.emit_arg(idx, |idx| Instruction::LoadConst { idx })
     }
 
@@ -2725,7 +2726,7 @@ impl Compiler {
 
     fn new_block(&mut self) -> ir::BlockIdx {
         let code = self.current_codeinfo();
-        let idx = ir::BlockIdx(code.blocks.len() as u32);
+        let idx = ir::BlockIdx(code.blocks.len().to_u32());
         code.blocks.push(ir::Block::default());
         idx
     }
@@ -2753,7 +2754,7 @@ impl Compiler {
     }
 
     fn get_source_line_number(&self) -> u32 {
-        self.current_source_location.row() as u32
+        self.current_source_location.row().to_u32()
     }
 
     fn push_qualified_path(&mut self, name: &str) {
@@ -2849,6 +2850,17 @@ fn compile_constant(value: &ast::Constant) -> ConstantData {
     }
 }
 
+// Note: Not a good practice in general. Keep this trait private only for compiler
+trait ToU32 {
+    fn to_u32(self) -> u32;
+}
+
+impl ToU32 for usize {
+    fn to_u32(self) -> u32 {
+        self.try_into().unwrap()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

From 7b84229db528d351848ac0f365c8cf6ce64039b7 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Thu, 2 Mar 2023 13:34:21 +0900
Subject: [PATCH 445/459] Add #![deny(clippy::cast_possible_truncation)]

---
 codegen/src/compile.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index 803ac803..f92e3b18 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -18,8 +18,8 @@ use num_complex::Complex64;
 use num_traits::ToPrimitive;
 use rustpython_ast as ast;
 use rustpython_compiler_core::{
-    self as bytecode, Arg as OpArgMarker, CodeObject, ConstantData, Instruction, Location, NameIdx,
-    OpArg, OpArgType,
+    self as bytecode, Arg as OpArgMarker, CodeObject, ConstantData, Instruction, Location, OpArg,
+    OpArgType,
 };
 use std::borrow::Cow;
 
@@ -547,7 +547,7 @@ impl Compiler {
                 NameUsage::Delete => Instruction::DeleteLocal,
             },
         };
-        self.emit_arg(idx as NameIdx, op);
+        self.emit_arg(idx.to_u32(), op);
 
         Ok(())
     }

From 86ce9e8ff3eaabec401ac46b0710f0d9a2d73dc4 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Thu, 2 Mar 2023 15:38:36 -0500
Subject: [PATCH 446/459] Add optional serde dependency

---
 core/Cargo.toml      | 1 +
 core/src/location.rs | 4 ++++
 parser/Cargo.toml    | 2 ++
 3 files changed, 7 insertions(+)

diff --git a/core/Cargo.toml b/core/Cargo.toml
index 03160356..4df8932c 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -13,5 +13,6 @@ bstr = { workspace = true }
 itertools = { workspace = true }
 num-bigint = { workspace = true }
 num-complex = { workspace = true }
+serde = { version = "1.0.133", optional = true, default-features = false }
 
 lz4_flex = "0.9.2"
diff --git a/core/src/location.rs b/core/src/location.rs
index 2a2d029d..a6e7d391 100644
--- a/core/src/location.rs
+++ b/core/src/location.rs
@@ -1,5 +1,9 @@
+#[cfg(feature = "serde")]
+use serde::{Deserialize, Serialize};
+
 /// Sourcecode location.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct Location {
     pub(super) row: u32,
     pub(super) column: u32,
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index 026616d3..4ff560a8 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -10,6 +10,7 @@ edition = "2021"
 
 [features]
 default = ["lalrpop"]  # removing this causes potential build failure
+serde = ["dep:serde", "rustpython-compiler-core/serde"]
 
 [build-dependencies]
 anyhow = { workspace = true }
@@ -33,6 +34,7 @@ unic-ucd-ident  = "0.9.0"
 lalrpop-util = "0.19.8"
 phf = "0.11.1"
 rustc-hash = "1.1.0"
+serde = { version = "1.0.133", optional = true, default-features = false }
 
 [dev-dependencies]
 insta = { workspace = true }

From d9f22c1b51390bac83e39ac6dcd0d837b6088aef Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Sat, 4 Mar 2023 12:42:05 -0500
Subject: [PATCH 447/459] Treat match and case as soft keywords in lambda
 assignments (#4623)

---
 parser/src/parser.rs                          |   2 +
 ...r__parser__tests__match_as_identifier.snap | 236 ++++++++++++++++++
 parser/src/soft_keywords.rs                   |   6 +-
 3 files changed, 243 insertions(+), 1 deletion(-)

diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 04875c44..6376e046 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -604,6 +604,8 @@ match match:
     case 1: pass
     case 2:
         pass
+match = lambda query: query == event
+print(match(12))
 "#,
             "<test>",
         )
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap b/parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap
index 4a3c8af3..2fb15a7f 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__match_as_identifier.snap
@@ -1671,4 +1671,240 @@ expression: parse_ast
             ],
         },
     },
+    Located {
+        location: Location {
+            row: 21,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 21,
+                column: 36,
+            },
+        ),
+        custom: (),
+        node: Assign {
+            targets: [
+                Located {
+                    location: Location {
+                        row: 21,
+                        column: 0,
+                    },
+                    end_location: Some(
+                        Location {
+                            row: 21,
+                            column: 5,
+                        },
+                    ),
+                    custom: (),
+                    node: Name {
+                        id: "match",
+                        ctx: Store,
+                    },
+                },
+            ],
+            value: Located {
+                location: Location {
+                    row: 21,
+                    column: 8,
+                },
+                end_location: Some(
+                    Location {
+                        row: 21,
+                        column: 36,
+                    },
+                ),
+                custom: (),
+                node: Lambda {
+                    args: Arguments {
+                        posonlyargs: [],
+                        args: [
+                            Located {
+                                location: Location {
+                                    row: 21,
+                                    column: 15,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 21,
+                                        column: 20,
+                                    },
+                                ),
+                                custom: (),
+                                node: ArgData {
+                                    arg: "query",
+                                    annotation: None,
+                                    type_comment: None,
+                                },
+                            },
+                        ],
+                        vararg: None,
+                        kwonlyargs: [],
+                        kw_defaults: [],
+                        kwarg: None,
+                        defaults: [],
+                    },
+                    body: Located {
+                        location: Location {
+                            row: 21,
+                            column: 22,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 21,
+                                column: 36,
+                            },
+                        ),
+                        custom: (),
+                        node: Compare {
+                            left: Located {
+                                location: Location {
+                                    row: 21,
+                                    column: 22,
+                                },
+                                end_location: Some(
+                                    Location {
+                                        row: 21,
+                                        column: 27,
+                                    },
+                                ),
+                                custom: (),
+                                node: Name {
+                                    id: "query",
+                                    ctx: Load,
+                                },
+                            },
+                            ops: [
+                                Eq,
+                            ],
+                            comparators: [
+                                Located {
+                                    location: Location {
+                                        row: 21,
+                                        column: 31,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 21,
+                                            column: 36,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "event",
+                                        ctx: Load,
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                },
+            },
+            type_comment: None,
+        },
+    },
+    Located {
+        location: Location {
+            row: 22,
+            column: 0,
+        },
+        end_location: Some(
+            Location {
+                row: 22,
+                column: 16,
+            },
+        ),
+        custom: (),
+        node: Expr {
+            value: Located {
+                location: Location {
+                    row: 22,
+                    column: 0,
+                },
+                end_location: Some(
+                    Location {
+                        row: 22,
+                        column: 16,
+                    },
+                ),
+                custom: (),
+                node: Call {
+                    func: Located {
+                        location: Location {
+                            row: 22,
+                            column: 0,
+                        },
+                        end_location: Some(
+                            Location {
+                                row: 22,
+                                column: 5,
+                            },
+                        ),
+                        custom: (),
+                        node: Name {
+                            id: "print",
+                            ctx: Load,
+                        },
+                    },
+                    args: [
+                        Located {
+                            location: Location {
+                                row: 22,
+                                column: 6,
+                            },
+                            end_location: Some(
+                                Location {
+                                    row: 22,
+                                    column: 15,
+                                },
+                            ),
+                            custom: (),
+                            node: Call {
+                                func: Located {
+                                    location: Location {
+                                        row: 22,
+                                        column: 6,
+                                    },
+                                    end_location: Some(
+                                        Location {
+                                            row: 22,
+                                            column: 11,
+                                        },
+                                    ),
+                                    custom: (),
+                                    node: Name {
+                                        id: "match",
+                                        ctx: Load,
+                                    },
+                                },
+                                args: [
+                                    Located {
+                                        location: Location {
+                                            row: 22,
+                                            column: 12,
+                                        },
+                                        end_location: Some(
+                                            Location {
+                                                row: 22,
+                                                column: 14,
+                                            },
+                                        ),
+                                        custom: (),
+                                        node: Constant {
+                                            value: Int(
+                                                12,
+                                            ),
+                                            kind: None,
+                                        },
+                                    },
+                                ],
+                                keywords: [],
+                            },
+                        },
+                    ],
+                    keywords: [],
+                },
+            },
+        },
+    },
 ]
diff --git a/parser/src/soft_keywords.rs b/parser/src/soft_keywords.rs
index 2613914f..b7b034f5 100644
--- a/parser/src/soft_keywords.rs
+++ b/parser/src/soft_keywords.rs
@@ -59,11 +59,15 @@ where
                     let mut nesting = 0;
                     let mut first = true;
                     let mut seen_colon = false;
+                    let mut seen_lambda = false;
                     while let Some(Ok((_, tok, _))) = self.underlying.peek() {
                         match tok {
                             Tok::Newline => break,
+                            Tok::Lambda if nesting == 0 => seen_lambda = true,
                             Tok::Colon if nesting == 0 => {
-                                if !first {
+                                if seen_lambda {
+                                    seen_lambda = false;
+                                } else if !first {
                                     seen_colon = true;
                                 }
                             }

From 18de25c8a6ff1385f523f03209892e7cdb697882 Mon Sep 17 00:00:00 2001
From: Noa <coolreader18@gmail.com>
Date: Fri, 13 Jan 2023 22:25:43 -0600
Subject: [PATCH 448/459] Rework frozen modules and directly deserialize to
 CodeObject<Literal>

---
 core/src/bytecode.rs | 167 +++++++++++++++++++++++++++----------------
 core/src/marshal.rs  |  33 ++++++---
 2 files changed, 129 insertions(+), 71 deletions(-)

diff --git a/core/src/bytecode.rs b/core/src/bytecode.rs
index 3b320318..c01c4b80 100644
--- a/core/src/bytecode.rs
+++ b/core/src/bytecode.rs
@@ -1,7 +1,6 @@
 //! Implement python as a virtual machine with bytecodes. This module
 //! implements bytecode structure.
 
-use crate::marshal::MarshalError;
 use crate::{marshal, Location};
 use bitflags::bitflags;
 use itertools::Itertools;
@@ -46,6 +45,19 @@ pub trait ConstantBag: Sized + Copy {
     fn make_name(&self, name: &str) -> <Self::Constant as Constant>::Name;
 }
 
+pub trait AsBag {
+    type Bag: ConstantBag;
+    #[allow(clippy::wrong_self_convention)]
+    fn as_bag(self) -> Self::Bag;
+}
+
+impl<Bag: ConstantBag> AsBag for Bag {
+    type Bag = Self;
+    fn as_bag(self) -> Self {
+        self
+    }
+}
+
 #[derive(Clone, Copy)]
 pub struct BasicBag;
 
@@ -1077,27 +1089,6 @@ impl<C: Constant> CodeObject<C> {
     }
 }
 
-impl CodeObject<ConstantData> {
-    /// Load a code object from bytes
-    pub fn from_bytes(data: &[u8]) -> Result<Self, MarshalError> {
-        use lz4_flex::block::DecompressError;
-        let raw_bincode = lz4_flex::decompress_size_prepended(data).map_err(|e| match e {
-            DecompressError::OutputTooSmall { .. } | DecompressError::ExpectedAnotherByte => {
-                MarshalError::Eof
-            }
-            _ => MarshalError::InvalidBytecode,
-        })?;
-        marshal::deserialize_code(&mut &raw_bincode[..], BasicBag)
-    }
-
-    /// Serialize this bytecode to bytes.
-    pub fn to_bytes(&self) -> Vec<u8> {
-        let mut data = Vec::new();
-        marshal::serialize_code(&mut data, self);
-        lz4_flex::compress_prepend_size(&data)
-    }
-}
-
 impl<C: Constant> fmt::Display for CodeObject<C> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         self.display_inner(f, false, 1)?;
@@ -1483,32 +1474,81 @@ impl<C: Constant> fmt::Debug for CodeObject<C> {
     }
 }
 
-/// A frozen module. Holds a code object and whether it is part of a package
-#[derive(Debug)]
-pub struct FrozenModule {
-    pub code: CodeObject<ConstantData>,
-    pub package: bool,
-}
-
 pub mod frozen_lib {
     use super::*;
-    use marshal::{Read, Write};
+    use marshal::{Read, ReadBorrowed, Write};
 
-    /// Decode a library to a iterable of frozen modules
-    pub fn decode_lib(bytes: &[u8]) -> FrozenModulesIter {
-        let data = lz4_flex::decompress_size_prepended(bytes).unwrap();
-        let mut data = marshal::Cursor { data, position: 0 };
-        let remaining = data.read_u32().unwrap();
-        FrozenModulesIter { remaining, data }
+    /// A frozen module. Holds a frozen code object and whether it is part of a package
+    #[derive(Copy, Clone)]
+    pub struct FrozenModule<B = &'static [u8]> {
+        pub code: FrozenCodeObject<B>,
+        pub package: bool,
     }
 
-    pub struct FrozenModulesIter {
+    #[derive(Copy, Clone)]
+    pub struct FrozenCodeObject<B> {
+        pub bytes: B,
+    }
+
+    impl<B: AsRef<[u8]>> FrozenCodeObject<B> {
+        /// Decode a frozen codeobject
+        #[inline]
+        pub fn decode<Bag: AsBag>(
+            &self,
+            bag: Bag,
+        ) -> CodeObject<<Bag::Bag as ConstantBag>::Constant> {
+            Self::_decode(self.bytes.as_ref(), bag.as_bag())
+        }
+        fn _decode<Bag: ConstantBag>(data: &[u8], bag: Bag) -> CodeObject<Bag::Constant> {
+            let decompressed = lz4_flex::decompress_size_prepended(data)
+                .expect("deserialize frozen CodeObject failed");
+            marshal::deserialize_code(&mut &decompressed[..], bag)
+                .expect("deserializing frozen CodeObject failed")
+        }
+    }
+
+    impl FrozenCodeObject<Vec<u8>> {
+        pub fn encode<C: Constant>(code: &CodeObject<C>) -> Self {
+            let mut data = Vec::new();
+            marshal::serialize_code(&mut data, code);
+            let bytes = lz4_flex::compress_prepend_size(&data);
+            FrozenCodeObject { bytes }
+        }
+    }
+
+    #[repr(transparent)]
+    pub struct FrozenLib<B: ?Sized = [u8]> {
+        pub bytes: B,
+    }
+
+    impl<B: AsRef<[u8]> + ?Sized> FrozenLib<B> {
+        pub const fn from_ref(b: &B) -> &FrozenLib<B> {
+            unsafe { &*(b as *const B as *const FrozenLib<B>) }
+        }
+
+        /// Decode a library to a iterable of frozen modules
+        pub fn decode(&self) -> FrozenModulesIter<'_> {
+            let mut data = self.bytes.as_ref();
+            let remaining = data.read_u32().unwrap();
+            FrozenModulesIter { remaining, data }
+        }
+    }
+
+    impl<'a, B: AsRef<[u8]> + ?Sized> IntoIterator for &'a FrozenLib<B> {
+        type Item = (&'a str, FrozenModule<&'a [u8]>);
+        type IntoIter = FrozenModulesIter<'a>;
+        fn into_iter(self) -> Self::IntoIter {
+            self.decode()
+        }
+    }
+
+    pub struct FrozenModulesIter<'a> {
         remaining: u32,
-        data: marshal::Cursor<Vec<u8>>,
+        data: &'a [u8],
     }
 
-    impl Iterator for FrozenModulesIter {
-        type Item = (String, FrozenModule);
+    impl<'a> Iterator for FrozenModulesIter<'a> {
+        type Item = (&'a str, FrozenModule<&'a [u8]>);
 
         fn next(&mut self) -> Option<Self::Item> {
             if self.remaining > 0 {
@@ -1524,31 +1564,37 @@ pub mod frozen_lib {
             (self.remaining as usize, Some(self.remaining as usize))
         }
     }
-    impl ExactSizeIterator for FrozenModulesIter {}
+    impl ExactSizeIterator for FrozenModulesIter<'_> {}
 
-    fn read_entry(rdr: &mut impl Read) -> Result<(String, FrozenModule), marshal::MarshalError> {
+    fn read_entry<'a>(
+        rdr: &mut &'a [u8],
+    ) -> Result<(&'a str, FrozenModule<&'a [u8]>), marshal::MarshalError> {
         let len = rdr.read_u32()?;
-        let name = rdr.read_str(len)?.to_owned();
-        let code = marshal::deserialize_code(rdr, BasicBag)?;
+        let name = rdr.read_str_borrow(len)?;
+        let len = rdr.read_u32()?;
+        let code_slice = rdr.read_slice_borrow(len)?;
+        let code = FrozenCodeObject { bytes: code_slice };
         let package = rdr.read_u8()? != 0;
         Ok((name, FrozenModule { code, package }))
     }
 
-    /// Encode the given iterator of frozen modules into a compressed vector of bytes
-    pub fn encode_lib<'a, I>(lib: I) -> Vec<u8>
-    where
-        I: IntoIterator<Item = (&'a str, &'a FrozenModule)>,
-        I::IntoIter: ExactSizeIterator + Clone,
-    {
-        let iter = lib.into_iter();
-        let mut data = Vec::new();
-        write_lib(&mut data, iter);
-        lz4_flex::compress_prepend_size(&data)
+    impl FrozenLib<Vec<u8>> {
+        /// Encode the given iterator of frozen modules into a compressed vector of bytes
+        pub fn encode<'a, I, B: AsRef<[u8]>>(lib: I) -> FrozenLib<Vec<u8>>
+        where
+            I: IntoIterator<Item = (&'a str, FrozenModule<B>)>,
+            I::IntoIter: ExactSizeIterator + Clone,
+        {
+            let iter = lib.into_iter();
+            let mut bytes = Vec::new();
+            write_lib(&mut bytes, iter);
+            Self { bytes }
+        }
     }
 
-    fn write_lib<'a>(
-        buf: &mut impl Write,
-        lib: impl ExactSizeIterator<Item = (&'a str, &'a FrozenModule)>,
+    fn write_lib<'a, B: AsRef<[u8]>>(
+        buf: &mut Vec<u8>,
+        lib: impl ExactSizeIterator<Item = (&'a str, FrozenModule<B>)>,
     ) {
         marshal::write_len(buf, lib.len());
         for (name, module) in lib {
@@ -1556,10 +1602,9 @@ pub mod frozen_lib {
         }
     }
 
-    fn write_entry(buf: &mut impl Write, name: &str, module: &FrozenModule) {
-        marshal::write_len(buf, name.len());
-        buf.write_slice(name.as_bytes());
-        marshal::serialize_code(buf, &module.code);
+    fn write_entry(buf: &mut Vec<u8>, name: &str, module: FrozenModule<impl AsRef<[u8]>>) {
+        marshal::write_vec(buf, name.as_bytes());
+        marshal::write_vec(buf, module.code.bytes.as_ref());
         buf.write_u8(module.package as u8);
     }
 }
diff --git a/core/src/marshal.rs b/core/src/marshal.rs
index 69d88793..e9f962fd 100644
--- a/core/src/marshal.rs
+++ b/core/src/marshal.rs
@@ -130,8 +130,21 @@ pub trait Read {
     }
 }
 
+pub(crate) trait ReadBorrowed<'a>: Read {
+    fn read_slice_borrow(&mut self, n: u32) -> Result<&'a [u8]>;
+    fn read_str_borrow(&mut self, len: u32) -> Result<&'a str> {
+        Ok(std::str::from_utf8(self.read_slice_borrow(len)?)?)
+    }
+}
+
 impl Read for &[u8] {
     fn read_slice(&mut self, n: u32) -> Result<&[u8]> {
+        self.read_slice_borrow(n)
+    }
+}
+
+impl<'a> ReadBorrowed<'a> for &'a [u8] {
+    fn read_slice_borrow(&mut self, n: u32) -> Result<&'a [u8]> {
         let data = self.get(..n as usize).ok_or(MarshalError::Eof)?;
         *self = &self[n as usize..];
         Ok(data)
@@ -474,6 +487,11 @@ pub(crate) fn write_len<W: Write>(buf: &mut W, len: usize) {
     buf.write_u32(len);
 }
 
+pub(crate) fn write_vec<W: Write>(buf: &mut W, slice: &[u8]) {
+    write_len(buf, slice.len());
+    buf.write_slice(slice);
+}
+
 pub fn serialize_value<W: Write, D: Dumpable>(
     buf: &mut W,
     constant: DumpableValue<'_, D>,
@@ -501,13 +519,11 @@ pub fn serialize_value<W: Write, D: Dumpable>(
         }
         DumpableValue::Str(s) => {
             buf.write_u8(Type::Unicode as u8);
-            write_len(buf, s.len());
-            buf.write_slice(s.as_bytes());
+            write_vec(buf, s.as_bytes());
         }
         DumpableValue::Bytes(b) => {
             buf.write_u8(Type::Bytes as u8);
-            write_len(buf, b.len());
-            buf.write_slice(b);
+            write_vec(buf, b);
         }
         DumpableValue::Code(c) => {
             buf.write_u8(Type::Code as u8);
@@ -580,14 +596,12 @@ pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>)
     buf.write_u32(code.arg_count);
     buf.write_u32(code.kwonlyarg_count);
 
-    write_len(buf, code.source_path.as_ref().len());
-    buf.write_slice(code.source_path.as_ref().as_bytes());
+    write_vec(buf, code.source_path.as_ref().as_bytes());
 
     buf.write_u32(code.first_line_number);
     buf.write_u32(code.max_stackdepth);
 
-    write_len(buf, code.obj_name.as_ref().len());
-    buf.write_slice(code.obj_name.as_ref().as_bytes());
+    write_vec(buf, code.obj_name.as_ref().as_bytes());
 
     let cell2arg = code.cell2arg.as_deref().unwrap_or(&[]);
     write_len(buf, cell2arg.len());
@@ -603,8 +617,7 @@ pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>)
     let mut write_names = |names: &[C::Name]| {
         write_len(buf, names.len());
         for name in names {
-            write_len(buf, name.as_ref().len());
-            buf.write_slice(name.as_ref().as_bytes());
+            write_vec(buf, name.as_ref().as_bytes());
         }
     };
 

From 5d2228999ba90f678bec05a3facf1927e64a40a7 Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Sun, 12 Mar 2023 14:46:10 -0400
Subject: [PATCH 449/459] Include Derive feature with optional Serde dependency

---
 core/Cargo.toml   | 2 +-
 parser/Cargo.toml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/Cargo.toml b/core/Cargo.toml
index 4df8932c..79622a95 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -13,6 +13,6 @@ bstr = { workspace = true }
 itertools = { workspace = true }
 num-bigint = { workspace = true }
 num-complex = { workspace = true }
-serde = { version = "1.0.133", optional = true, default-features = false }
+serde = { version = "1.0.133", optional = true, default-features = false, features = ["derive"] }
 
 lz4_flex = "0.9.2"
diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index 4ff560a8..1cb80746 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -34,7 +34,7 @@ unic-ucd-ident  = "0.9.0"
 lalrpop-util = "0.19.8"
 phf = "0.11.1"
 rustc-hash = "1.1.0"
-serde = { version = "1.0.133", optional = true, default-features = false }
+serde = { version = "1.0.133", optional = true, default-features = false, features = ["derive"] }
 
 [dev-dependencies]
 insta = { workspace = true }

From 70f8e04af3eecfff4a52720ca35594f502def58c Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Thu, 16 Mar 2023 21:27:02 +0900
Subject: [PATCH 450/459] Update cspell for compiler

---
 ast/src/unparse.rs                            |  30 ++---
 codegen/src/compile.rs                        | 102 +++++++--------
 codegen/src/ir.rs                             |  17 +--
 ...pile__tests__nested_double_async_with.snap |   4 +-
 codegen/src/symboltable.rs                    |  22 ++--
 core/src/bytecode.rs                          | 116 +++++++++---------
 core/src/location.rs                          |   2 +-
 parser/build.rs                               |   2 +-
 parser/src/lexer.rs                           |  41 ++++---
 parser/src/parser.rs                          |  12 +-
 ...python_parser__context__tests__assign.snap |  42 -------
 ...unction__tests__function_kw_only_args.snap |   1 -
 ...__function_kw_only_args_with_defaults.snap |   1 -
 ...er__parser__tests__parse_bool_op_and.snap} |   0
 ...ser__parser__tests__parse_bool_op_or.snap} |   0
 ...on_parser__parser__tests__parse_empty.snap |   2 +-
 ...hon_parser__parser__tests__star_index.snap |  78 ++++++------
 ..._fstring_parse_self_documenting_base.snap} |   1 -
 ...ing_parse_self_documenting_base_more.snap} |   1 -
 ...string_parse_self_documenting_format.snap} |   1 -
 ...r__string__tests__parse_empty_fstring.snap |   1 -
 ..._parser__string__tests__parse_fstring.snap |   1 -
 ...__string__tests__parse_fstring_equals.snap |   1 -
 ...ing__tests__parse_fstring_nested_spec.snap |   1 -
 ...ring__tests__parse_fstring_not_equals.snap |   1 -
 ..._tests__parse_fstring_not_nested_spec.snap |   1 -
 ...s__parse_fstring_self_doc_prec_space.snap} |   1 -
 ...arse_fstring_self_doc_trailing_space.snap} |   1 -
 ...ring__tests__parse_fstring_yield_expr.snap |   1 -
 parser/src/soft_keywords.rs                   |   2 +-
 parser/src/string.rs                          |  30 ++---
 src/lib.rs                                    |   2 +-
 32 files changed, 234 insertions(+), 284 deletions(-)
 delete mode 100644 parser/src/snapshots/rustpython_parser__context__tests__assign.snap
 rename parser/src/snapshots/{rustpython_parser__parser__tests__parse_boolop_and.snap => rustpython_parser__parser__tests__parse_bool_op_and.snap} (100%)
 rename parser/src/snapshots/{rustpython_parser__parser__tests__parse_boolop_or.snap => rustpython_parser__parser__tests__parse_bool_op_or.snap} (100%)
 rename parser/src/snapshots/{rustpython_parser__string__tests__fstring_parse_selfdocumenting_base.snap => rustpython_parser__string__tests__fstring_parse_self_documenting_base.snap} (98%)
 rename parser/src/snapshots/{rustpython_parser__string__tests__fstring_parse_selfdocumenting_base_more.snap => rustpython_parser__string__tests__fstring_parse_self_documenting_base_more.snap} (99%)
 rename parser/src/snapshots/{rustpython_parser__string__tests__fstring_parse_selfdocumenting_format.snap => rustpython_parser__string__tests__fstring_parse_self_documenting_format.snap} (99%)
 rename parser/src/snapshots/{rustpython_parser__string__tests__parse_fstring_selfdoc_prec_space.snap => rustpython_parser__string__tests__parse_fstring_self_doc_prec_space.snap} (98%)
 rename parser/src/snapshots/{rustpython_parser__string__tests__parse_fstring_selfdoc_trailing_space.snap => rustpython_parser__string__tests__parse_fstring_self_doc_trailing_space.snap} (98%)

diff --git a/ast/src/unparse.rs b/ast/src/unparse.rs
index b8b34209..081c2a92 100644
--- a/ast/src/unparse.rs
+++ b/ast/src/unparse.rs
@@ -48,10 +48,10 @@ impl<'a> Unparser<'a> {
     }
 
     fn unparse_expr<U>(&mut self, ast: &Expr<U>, level: u8) -> fmt::Result {
-        macro_rules! opprec {
-            ($opty:ident, $x:expr, $enu:path, $($var:ident($op:literal, $prec:ident)),*$(,)?) => {
+        macro_rules! op_prec {
+            ($op_ty:ident, $x:expr, $enu:path, $($var:ident($op:literal, $prec:ident)),*$(,)?) => {
                 match $x {
-                    $(<$enu>::$var => (opprec!(@space $opty, $op), precedence::$prec),)*
+                    $(<$enu>::$var => (op_prec!(@space $op_ty, $op), precedence::$prec),)*
                 }
             };
             (@space bin, $op:literal) => {
@@ -72,7 +72,7 @@ impl<'a> Unparser<'a> {
         }
         match &ast.node {
             ExprKind::BoolOp { op, values } => {
-                let (op, prec) = opprec!(bin, op, Boolop, And("and", AND), Or("or", OR));
+                let (op, prec) = op_prec!(bin, op, Boolop, And("and", AND), Or("or", OR));
                 group_if!(prec, {
                     let mut first = true;
                     for val in values {
@@ -89,8 +89,8 @@ impl<'a> Unparser<'a> {
                 })
             }
             ExprKind::BinOp { left, op, right } => {
-                let rassoc = matches!(op, Operator::Pow);
-                let (op, prec) = opprec!(
+                let right_associative = matches!(op, Operator::Pow);
+                let (op, prec) = op_prec!(
                     bin,
                     op,
                     Operator,
@@ -109,13 +109,13 @@ impl<'a> Unparser<'a> {
                     FloorDiv("//", TERM),
                 );
                 group_if!(prec, {
-                    self.unparse_expr(left, prec + rassoc as u8)?;
+                    self.unparse_expr(left, prec + right_associative as u8)?;
                     self.p(op)?;
-                    self.unparse_expr(right, prec + !rassoc as u8)?;
+                    self.unparse_expr(right, prec + !right_associative as u8)?;
                 })
             }
             ExprKind::UnaryOp { op, operand } => {
-                let (op, prec) = opprec!(
+                let (op, prec) = op_prec!(
                     un,
                     op,
                     crate::Unaryop,
@@ -131,8 +131,8 @@ impl<'a> Unparser<'a> {
             }
             ExprKind::Lambda { args, body } => {
                 group_if!(precedence::TEST, {
-                    let npos = args.args.len() + args.posonlyargs.len();
-                    self.p(if npos > 0 { "lambda " } else { "lambda" })?;
+                    let pos = args.args.len() + args.posonlyargs.len();
+                    self.p(if pos > 0 { "lambda " } else { "lambda" })?;
                     self.unparse_args(args)?;
                     write!(self, ": {}", **body)?;
                 })
@@ -260,7 +260,7 @@ impl<'a> Unparser<'a> {
                     [],
                 ) = (&**args, &**keywords)
                 {
-                    // make sure a single genexp doesn't get double parens
+                    // make sure a single genexpr doesn't get double parens
                     self.unparse_expr(elt, precedence::TEST)?;
                     self.unparse_comp(generators)?;
                 } else {
@@ -287,7 +287,7 @@ impl<'a> Unparser<'a> {
                 conversion,
                 format_spec,
             } => self.unparse_formatted(value, *conversion, format_spec.as_deref())?,
-            ExprKind::JoinedStr { values } => self.unparse_joinedstr(values, false)?,
+            ExprKind::JoinedStr { values } => self.unparse_joined_str(values, false)?,
             ExprKind::Constant { value, kind } => {
                 if let Some(kind) = kind {
                     self.p(kind)?;
@@ -490,7 +490,7 @@ impl<'a> Unparser<'a> {
                     unreachable!()
                 }
             }
-            ExprKind::JoinedStr { values } => self.unparse_joinedstr(values, is_spec),
+            ExprKind::JoinedStr { values } => self.unparse_joined_str(values, is_spec),
             ExprKind::FormattedValue {
                 value,
                 conversion,
@@ -505,7 +505,7 @@ impl<'a> Unparser<'a> {
         self.p(&s)
     }
 
-    fn unparse_joinedstr<U>(&mut self, values: &[Expr<U>], is_spec: bool) -> fmt::Result {
+    fn unparse_joined_str<U>(&mut self, values: &[Expr<U>], is_spec: bool) -> fmt::Result {
         if is_spec {
             self.unparse_fstring_body(values, is_spec)
         } else {
diff --git a/codegen/src/compile.rs b/codegen/src/compile.rs
index f92e3b18..4d940961 100644
--- a/codegen/src/compile.rs
+++ b/codegen/src/compile.rs
@@ -189,14 +189,14 @@ macro_rules! emit {
     ($c:expr, Instruction::$op:ident { $arg:ident$(,)? }$(,)?) => {
         $c.emit_arg($arg, |x| Instruction::$op { $arg: x })
     };
-    ($c:expr, Instruction::$op:ident { $arg:ident : $argval:expr $(,)? }$(,)?) => {
-        $c.emit_arg($argval, |x| Instruction::$op { $arg: x })
+    ($c:expr, Instruction::$op:ident { $arg:ident : $arg_val:expr $(,)? }$(,)?) => {
+        $c.emit_arg($arg_val, |x| Instruction::$op { $arg: x })
     };
-    ($c:expr, Instruction::$op:ident( $argval:expr $(,)? )$(,)?) => {
-        $c.emit_arg($argval, Instruction::$op)
+    ($c:expr, Instruction::$op:ident( $arg_val:expr $(,)? )$(,)?) => {
+        $c.emit_arg($arg_val, Instruction::$op)
     };
     ($c:expr, Instruction::$op:ident$(,)?) => {
-        $c.emit_noarg(Instruction::$op)
+        $c.emit_no_arg(Instruction::$op)
     };
 }
 
@@ -331,7 +331,7 @@ impl Compiler {
         cache: impl FnOnce(&mut ir::CodeInfo) -> &mut IndexSet<String>,
     ) -> bytecode::NameIdx {
         let name = self.mangle(name);
-        let cache = cache(self.current_codeinfo());
+        let cache = cache(self.current_code_info());
         cache
             .get_index_of(name.as_ref())
             .unwrap_or_else(|| cache.insert_full(name.into_owned()).0)
@@ -821,7 +821,7 @@ impl Compiler {
                     Some(v) => {
                         if self.ctx.func == FunctionContext::AsyncFunction
                             && self
-                                .current_codeinfo()
+                                .current_code_info()
                                 .flags
                                 .contains(bytecode::CodeFlags::IS_GENERATOR)
                         {
@@ -929,12 +929,12 @@ impl Compiler {
             );
         }
 
-        let mut funcflags = bytecode::MakeFunctionFlags::empty();
+        let mut func_flags = bytecode::MakeFunctionFlags::empty();
         if have_defaults {
-            funcflags |= bytecode::MakeFunctionFlags::DEFAULTS;
+            func_flags |= bytecode::MakeFunctionFlags::DEFAULTS;
         }
         if !args.kw_defaults.is_empty() {
-            funcflags |= bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS;
+            func_flags |= bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS;
         }
 
         self.push_output(
@@ -954,15 +954,15 @@ impl Compiler {
         }
 
         if let Some(name) = args.vararg.as_deref() {
-            self.current_codeinfo().flags |= bytecode::CodeFlags::HAS_VARARGS;
+            self.current_code_info().flags |= bytecode::CodeFlags::HAS_VARARGS;
             self.varname(&name.node.arg)?;
         }
         if let Some(name) = args.kwarg.as_deref() {
-            self.current_codeinfo().flags |= bytecode::CodeFlags::HAS_VARKEYWORDS;
+            self.current_code_info().flags |= bytecode::CodeFlags::HAS_VARKEYWORDS;
             self.varname(&name.node.arg)?;
         }
 
-        Ok(funcflags)
+        Ok(func_flags)
     }
 
     fn prepare_decorators(&mut self, decorator_list: &[ast::Expr]) -> CompileResult<()> {
@@ -1133,8 +1133,8 @@ impl Compiler {
         // Create bytecode for this function:
 
         self.prepare_decorators(decorator_list)?;
-        let mut funcflags = self.enter_function(name, args)?;
-        self.current_codeinfo()
+        let mut func_flags = self.enter_function(name, args)?;
+        self.current_code_info()
             .flags
             .set(bytecode::CodeFlags::IS_COROUTINE, is_async);
 
@@ -1157,7 +1157,7 @@ impl Compiler {
 
         let (doc_str, body) = split_doc(body);
 
-        self.current_codeinfo()
+        self.current_code_info()
             .constants
             .insert_full(ConstantData::None);
 
@@ -1210,7 +1210,7 @@ impl Compiler {
         }
 
         if num_annotations > 0 {
-            funcflags |= bytecode::MakeFunctionFlags::ANNOTATIONS;
+            func_flags |= bytecode::MakeFunctionFlags::ANNOTATIONS;
             emit!(
                 self,
                 Instruction::BuildMap {
@@ -1220,7 +1220,7 @@ impl Compiler {
         }
 
         if self.build_closure(&code) {
-            funcflags |= bytecode::MakeFunctionFlags::CLOSURE;
+            func_flags |= bytecode::MakeFunctionFlags::CLOSURE;
         }
 
         self.emit_constant(ConstantData::Code {
@@ -1231,7 +1231,7 @@ impl Compiler {
         });
 
         // Turn code object into function object:
-        emit!(self, Instruction::MakeFunction(funcflags));
+        emit!(self, Instruction::MakeFunction(func_flags));
 
         emit!(self, Instruction::Duplicate);
         self.load_docstring(doc_str);
@@ -1388,10 +1388,10 @@ impl Compiler {
         self.qualified_path.append(global_path_prefix.as_mut());
         self.ctx = prev_ctx;
 
-        let mut funcflags = bytecode::MakeFunctionFlags::empty();
+        let mut func_flags = bytecode::MakeFunctionFlags::empty();
 
         if self.build_closure(&code) {
-            funcflags |= bytecode::MakeFunctionFlags::CLOSURE;
+            func_flags |= bytecode::MakeFunctionFlags::CLOSURE;
         }
 
         self.emit_constant(ConstantData::Code {
@@ -1402,7 +1402,7 @@ impl Compiler {
         });
 
         // Turn code object into function object:
-        emit!(self, Instruction::MakeFunction(funcflags));
+        emit!(self, Instruction::MakeFunction(func_flags));
 
         self.emit_constant(ConstantData::Str {
             value: name.to_owned(),
@@ -1417,7 +1417,7 @@ impl Compiler {
     }
 
     fn load_docstring(&mut self, doc_str: Option<String>) {
-        // TODO: __doc__ must be default None and no bytecodes unless it is Some
+        // TODO: __doc__ must be default None and no bytecode unless it is Some
         // Duplicate top of stack (the function or class object)
 
         // Doc string value:
@@ -1603,12 +1603,12 @@ impl Compiler {
         &mut self,
         left: &ast::Expr,
         ops: &[ast::Cmpop],
-        vals: &[ast::Expr],
+        exprs: &[ast::Expr],
     ) -> CompileResult<()> {
         assert!(!ops.is_empty());
-        assert_eq!(vals.len(), ops.len());
+        assert_eq!(exprs.len(), ops.len());
         let (last_op, mid_ops) = ops.split_last().unwrap();
-        let (last_val, mid_vals) = vals.split_last().unwrap();
+        let (last_val, mid_exprs) = exprs.split_last().unwrap();
 
         use bytecode::ComparisonOperator::*;
         use bytecode::TestOperator::*;
@@ -1626,7 +1626,7 @@ impl Compiler {
         };
 
         // a == b == c == d
-        // compile into (pseudocode):
+        // compile into (pseudo code):
         // result = a == b
         // if result:
         //   result = b == c
@@ -1636,7 +1636,7 @@ impl Compiler {
         // initialize lhs outside of loop
         self.compile_expression(left)?;
 
-        let end_blocks = if mid_vals.is_empty() {
+        let end_blocks = if mid_exprs.is_empty() {
             None
         } else {
             let break_block = self.new_block();
@@ -1645,7 +1645,7 @@ impl Compiler {
         };
 
         // for all comparisons except the last (as the last one doesn't need a conditional jump)
-        for (op, val) in mid_ops.iter().zip(mid_vals) {
+        for (op, val) in mid_ops.iter().zip(mid_exprs) {
             self.compile_expression(val)?;
             // store rhs for the next comparison in chain
             emit!(self, Instruction::Duplicate);
@@ -2206,7 +2206,7 @@ impl Compiler {
                 let prev_ctx = self.ctx;
 
                 let name = "<lambda>".to_owned();
-                let mut funcflags = self.enter_function(&name, args)?;
+                let mut func_flags = self.enter_function(&name, args)?;
 
                 self.ctx = CompileContext {
                     loop_data: Option::None,
@@ -2214,7 +2214,7 @@ impl Compiler {
                     func: FunctionContext::Function,
                 };
 
-                self.current_codeinfo()
+                self.current_code_info()
                     .constants
                     .insert_full(ConstantData::None);
 
@@ -2222,14 +2222,14 @@ impl Compiler {
                 emit!(self, Instruction::ReturnValue);
                 let code = self.pop_code_object();
                 if self.build_closure(&code) {
-                    funcflags |= bytecode::MakeFunctionFlags::CLOSURE;
+                    func_flags |= bytecode::MakeFunctionFlags::CLOSURE;
                 }
                 self.emit_constant(ConstantData::Code {
                     code: Box::new(code),
                 });
                 self.emit_constant(ConstantData::Str { value: name });
                 // Turn code object into function object:
-                emit!(self, Instruction::MakeFunction(funcflags));
+                emit!(self, Instruction::MakeFunction(func_flags));
 
                 self.ctx = prev_ctx;
             }
@@ -2345,24 +2345,24 @@ impl Compiler {
     fn compile_keywords(&mut self, keywords: &[ast::Keyword]) -> CompileResult<()> {
         let mut size = 0;
         let groupby = keywords.iter().group_by(|e| e.node.arg.is_none());
-        for (is_unpacking, subkeywords) in &groupby {
+        for (is_unpacking, sub_keywords) in &groupby {
             if is_unpacking {
-                for keyword in subkeywords {
+                for keyword in sub_keywords {
                     self.compile_expression(&keyword.node.value)?;
                     size += 1;
                 }
             } else {
-                let mut subsize = 0;
-                for keyword in subkeywords {
+                let mut sub_size = 0;
+                for keyword in sub_keywords {
                     if let Some(name) = &keyword.node.arg {
                         self.emit_constant(ConstantData::Str {
                             value: name.to_owned(),
                         });
                         self.compile_expression(&keyword.node.value)?;
-                        subsize += 1;
+                        sub_size += 1;
                     }
                 }
-                emit!(self, Instruction::BuildMap { size: subsize });
+                emit!(self, Instruction::BuildMap { size: sub_size });
                 size += 1;
             }
         }
@@ -2634,9 +2634,9 @@ impl Compiler {
 
         self.ctx = prev_ctx;
 
-        let mut funcflags = bytecode::MakeFunctionFlags::empty();
+        let mut func_flags = bytecode::MakeFunctionFlags::empty();
         if self.build_closure(&code) {
-            funcflags |= bytecode::MakeFunctionFlags::CLOSURE;
+            func_flags |= bytecode::MakeFunctionFlags::CLOSURE;
         }
 
         // List comprehension code:
@@ -2650,7 +2650,7 @@ impl Compiler {
         });
 
         // Turn code object into function object:
-        emit!(self, Instruction::MakeFunction(funcflags));
+        emit!(self, Instruction::MakeFunction(func_flags));
 
         // Evaluate iterated item:
         self.compile_expression(&generators[0].iter)?;
@@ -2694,7 +2694,7 @@ impl Compiler {
         });
     }
 
-    fn emit_noarg(&mut self, ins: Instruction) {
+    fn emit_no_arg(&mut self, ins: Instruction) {
         self._emit(ins, OpArg::null(), ir::BlockIdx::NULL)
     }
 
@@ -2710,29 +2710,29 @@ impl Compiler {
     // fn block_done()
 
     fn emit_constant(&mut self, constant: ConstantData) {
-        let info = self.current_codeinfo();
+        let info = self.current_code_info();
         let idx = info.constants.insert_full(constant).0.to_u32();
         self.emit_arg(idx, |idx| Instruction::LoadConst { idx })
     }
 
-    fn current_codeinfo(&mut self) -> &mut ir::CodeInfo {
+    fn current_code_info(&mut self) -> &mut ir::CodeInfo {
         self.code_stack.last_mut().expect("no code on stack")
     }
 
     fn current_block(&mut self) -> &mut ir::Block {
-        let info = self.current_codeinfo();
+        let info = self.current_code_info();
         &mut info.blocks[info.current_block]
     }
 
     fn new_block(&mut self) -> ir::BlockIdx {
-        let code = self.current_codeinfo();
+        let code = self.current_code_info();
         let idx = ir::BlockIdx(code.blocks.len().to_u32());
         code.blocks.push(ir::Block::default());
         idx
     }
 
     fn switch_to_block(&mut self, block: ir::BlockIdx) {
-        let code = self.current_codeinfo();
+        let code = self.current_code_info();
         let prev = code.current_block;
         assert_eq!(
             code.blocks[block].next,
@@ -2762,7 +2762,7 @@ impl Compiler {
     }
 
     fn mark_generator(&mut self) {
-        self.current_codeinfo().flags |= bytecode::CodeFlags::IS_GENERATOR
+        self.current_code_info().flags |= bytecode::CodeFlags::IS_GENERATOR
     }
 }
 
@@ -2882,7 +2882,7 @@ mod tests {
         ($value:expr) => {
             insta::assert_snapshot!(
                 insta::internals::AutoName,
-                $value.display_expand_codeobjects().to_string(),
+                $value.display_expand_code_objects().to_string(),
                 stringify!($value)
             )
         };
@@ -2925,7 +2925,7 @@ if (True and False) or (False and True):
 for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):
     with self.subTest(type=type(stop_exc)):
         try:
-            async with woohoo():
+            async with egg():
                 raise stop_exc
         except Exception as ex:
             self.assertIs(ex, stop_exc)
diff --git a/codegen/src/ir.rs b/codegen/src/ir.rs
index 2658028d..dc7e2e74 100644
--- a/codegen/src/ir.rs
+++ b/codegen/src/ir.rs
@@ -45,6 +45,7 @@ pub struct InstructionInfo {
     pub location: Location,
 }
 
+// spell-checker:ignore petgraph
 // TODO: look into using petgraph for handling blocks and stuff? it's heavier than this, but it
 // might enable more analysis/optimizations
 #[derive(Debug)]
@@ -223,12 +224,12 @@ impl CodeInfo {
     fn max_stackdepth(&self) -> u32 {
         let mut maxdepth = 0u32;
         let mut stack = Vec::with_capacity(self.blocks.len());
-        let mut startdepths = vec![u32::MAX; self.blocks.len()];
-        startdepths[0] = 0;
+        let mut start_depths = vec![u32::MAX; self.blocks.len()];
+        start_depths[0] = 0;
         stack.push(BlockIdx(0));
         const DEBUG: bool = false;
         'process_blocks: while let Some(block) = stack.pop() {
-            let mut depth = startdepths[block.idx()];
+            let mut depth = start_depths[block.idx()];
             if DEBUG {
                 eprintln!("===BLOCK {}===", block.0);
             }
@@ -266,14 +267,14 @@ impl CodeInfo {
                     if target_depth > maxdepth {
                         maxdepth = target_depth
                     }
-                    stackdepth_push(&mut stack, &mut startdepths, i.target, target_depth);
+                    stackdepth_push(&mut stack, &mut start_depths, i.target, target_depth);
                 }
                 depth = new_depth;
                 if instr.unconditional_branch() {
                     continue 'process_blocks;
                 }
             }
-            stackdepth_push(&mut stack, &mut startdepths, block.next, depth);
+            stackdepth_push(&mut stack, &mut start_depths, block.next, depth);
         }
         if DEBUG {
             eprintln!("DONE: {maxdepth}");
@@ -293,7 +294,7 @@ impl InstrDisplayContext for CodeInfo {
     fn get_varname(&self, i: usize) -> &str {
         self.varname_cache[i].as_ref()
     }
-    fn get_cellname(&self, i: usize) -> &str {
+    fn get_cell_name(&self, i: usize) -> &str {
         self.cellvar_cache
             .get_index(i)
             .unwrap_or_else(|| &self.freevar_cache[i - self.cellvar_cache.len()])
@@ -303,11 +304,11 @@ impl InstrDisplayContext for CodeInfo {
 
 fn stackdepth_push(
     stack: &mut Vec<BlockIdx>,
-    startdepths: &mut [u32],
+    start_depths: &mut [u32],
     target: BlockIdx,
     depth: u32,
 ) {
-    let block_depth = &mut startdepths[target.idx()];
+    let block_depth = &mut start_depths[target.idx()];
     if *block_depth == u32::MAX || depth > *block_depth {
         *block_depth = depth;
         stack.push(target);
diff --git a/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap b/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap
index 0563b38e..dcc6f4c2 100644
--- a/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap
+++ b/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap
@@ -1,6 +1,6 @@
 ---
 source: compiler/codegen/src/compile.rs
-expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n    with self.subTest(type=type(stop_exc)):\n        try:\n            async with woohoo():\n                raise stop_exc\n        except Exception as ex:\n            self.assertIs(ex, stop_exc)\n        else:\n            self.fail(f'{stop_exc} was suppressed')\n\")"
+expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n    with self.subTest(type=type(stop_exc)):\n        try:\n            async with egg():\n                raise stop_exc\n        except Exception as ex:\n            self.assertIs(ex, stop_exc)\n        else:\n            self.fail(f'{stop_exc} was suppressed')\n\")"
 ---
   1           0 SetupLoop
               1 LoadNameAny          (0, StopIteration)
@@ -26,7 +26,7 @@ expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyn
 
   3          20 SetupExcept          (40)
 
-  4          21 LoadNameAny          (6, woohoo)
+  4          21 LoadNameAny          (6, egg)
              22 CallFunctionPositional(0)
              23 BeforeAsyncWith
              24 GetAwaitable
diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index f2a7c42b..258d3577 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -16,7 +16,7 @@ use rustpython_ast as ast;
 use rustpython_compiler_core::Location;
 use std::{borrow::Cow, fmt};
 
-/// Captures all symbols in the current scope, and has a list of subscopes in this scope.
+/// Captures all symbols in the current scope, and has a list of sub-scopes in this scope.
 #[derive(Clone)]
 pub struct SymbolTable {
     /// The name of this symbol table. Often the name of the class or function.
@@ -25,7 +25,7 @@ pub struct SymbolTable {
     /// The type of symbol table
     pub typ: SymbolTableType,
 
-    /// The line number in the sourcecode where this symboltable begins.
+    /// The line number in the source code where this symboltable begins.
     pub line_number: usize,
 
     // Return True if the block is a nested class or function
@@ -34,7 +34,7 @@ pub struct SymbolTable {
     /// A set of symbols present on this scope level.
     pub symbols: IndexMap<String, Symbol>,
 
-    /// A list of subscopes in the order as found in the
+    /// A list of sub-scopes in the order as found in the
     /// AST nodes.
     pub sub_tables: Vec<SymbolTable>,
 }
@@ -107,7 +107,7 @@ bitflags! {
         // this is required to correct the scope in the analysis.
         const ASSIGNED_IN_COMPREHENSION = 0x040;
         // indicates that the symbol is used a bound iterator variable. We distinguish this case
-        // from normal assignment to detect unallowed re-assignment to iterator variables.
+        // from normal assignment to detect disallowed re-assignment to iterator variables.
         const ITER = 0x080;
         /// indicates that the symbol is a free variable in a class method from the scope that the
         /// class is defined in, e.g.:
@@ -531,7 +531,7 @@ enum SymbolUsage {
     AnnotationAssigned,
     Parameter,
     AnnotationParameter,
-    AssignedNamedExprInCompr,
+    AssignedNamedExprInComprehension,
     Iter,
 }
 
@@ -741,7 +741,7 @@ impl SymbolTableBuilder {
             Import { names } | ImportFrom { names, .. } => {
                 for name in names {
                     if let Some(alias) = &name.node.asname {
-                        // `import mymodule as myalias`
+                        // `import my_module as my_alias`
                         self.register_name(alias, SymbolUsage::Imported, location)?;
                     } else {
                         // `import module`
@@ -1042,7 +1042,11 @@ impl SymbolTableBuilder {
                 if let Name { id, .. } = &target.node {
                     let table = self.tables.last().unwrap();
                     if table.typ == SymbolTableType::Comprehension {
-                        self.register_name(id, SymbolUsage::AssignedNamedExprInCompr, location)?;
+                        self.register_name(
+                            id,
+                            SymbolUsage::AssignedNamedExprInComprehension,
+                            location,
+                        )?;
                     } else {
                         // omit one recursion. When the handling of an store changes for
                         // Identifiers this needs adapted - more forward safe would be
@@ -1256,7 +1260,7 @@ impl SymbolTableBuilder {
             SymbolUsage::Assigned => {
                 flags.insert(SymbolFlags::ASSIGNED);
             }
-            SymbolUsage::AssignedNamedExprInCompr => {
+            SymbolUsage::AssignedNamedExprInComprehension => {
                 flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::ASSIGNED_IN_COMPREHENSION);
             }
             SymbolUsage::Global => {
@@ -1273,7 +1277,7 @@ impl SymbolTableBuilder {
         // and even more checking
         // it is not allowed to assign to iterator variables (by named expressions)
         if flags.contains(SymbolFlags::ITER | SymbolFlags::ASSIGNED)
-        /*&& symbol.is_assign_namedexpr_in_comprehension*/
+        /*&& symbol.is_assign_named_expr_in_comprehension*/
         {
             return Err(SymbolTableError {
                 error:
diff --git a/core/src/bytecode.rs b/core/src/bytecode.rs
index c01c4b80..a522d3fb 100644
--- a/core/src/bytecode.rs
+++ b/core/src/bytecode.rs
@@ -1,4 +1,4 @@
-//! Implement python as a virtual machine with bytecodes. This module
+//! Implement python as a virtual machine with bytecode. This module
 //! implements bytecode structure.
 
 use crate::{marshal, Location};
@@ -85,7 +85,7 @@ impl ConstantBag for BasicBag {
 }
 
 /// Primary container of a single code object. Each python function has
-/// a codeobject. Also a module has a codeobject.
+/// a code object. Also a module has a code object.
 #[derive(Clone)]
 pub struct CodeObject<C: Constant = ConstantData> {
     pub instructions: Box<[CodeUnit]>,
@@ -147,7 +147,7 @@ impl fmt::Debug for OpArgByte {
     }
 }
 
-/// a full 32-bit oparg, including any possible ExtendedArg extension
+/// a full 32-bit op_arg, including any possible ExtendedArg extension
 #[derive(Copy, Clone, Debug)]
 #[repr(transparent)]
 pub struct OpArg(pub u32);
@@ -156,7 +156,7 @@ impl OpArg {
         OpArg(0)
     }
 
-    /// Returns how many CodeUnits a instruction with this oparg will be encoded as
+    /// Returns how many CodeUnits a instruction with this op_arg will be encoded as
     #[inline]
     pub fn instr_size(self) -> usize {
         (self.0 > 0xff) as usize + (self.0 > 0xff_ff) as usize + (self.0 > 0xff_ff_ff) as usize + 1
@@ -204,46 +204,46 @@ impl OpArgState {
 }
 
 pub trait OpArgType: Copy {
-    fn from_oparg(x: u32) -> Option<Self>;
-    fn to_oparg(self) -> u32;
+    fn from_op_arg(x: u32) -> Option<Self>;
+    fn to_op_arg(self) -> u32;
 }
 
 impl OpArgType for u32 {
     #[inline(always)]
-    fn from_oparg(x: u32) -> Option<Self> {
+    fn from_op_arg(x: u32) -> Option<Self> {
         Some(x)
     }
     #[inline(always)]
-    fn to_oparg(self) -> u32 {
+    fn to_op_arg(self) -> u32 {
         self
     }
 }
 
 impl OpArgType for bool {
     #[inline(always)]
-    fn from_oparg(x: u32) -> Option<Self> {
+    fn from_op_arg(x: u32) -> Option<Self> {
         Some(x != 0)
     }
     #[inline(always)]
-    fn to_oparg(self) -> u32 {
+    fn to_op_arg(self) -> u32 {
         self as u32
     }
 }
 
-macro_rules! oparg_enum {
-    ($(#[$attr:meta])* $vis:vis enum $name:ident { $($(#[$var_attr:meta])* $var:ident = $discr:literal,)* }) => {
+macro_rules! op_arg_enum {
+    ($(#[$attr:meta])* $vis:vis enum $name:ident { $($(#[$var_attr:meta])* $var:ident = $value:literal,)* }) => {
         $(#[$attr])*
         $vis enum $name {
-            $($(#[$var_attr])* $var = $discr,)*
+            $($(#[$var_attr])* $var = $value,)*
         }
 
         impl OpArgType for $name {
-            fn to_oparg(self) -> u32 {
+            fn to_op_arg(self) -> u32 {
                 self as u32
             }
-            fn from_oparg(x: u32) -> Option<Self> {
+            fn from_op_arg(x: u32) -> Option<Self> {
                 Some(match u8::try_from(x).ok()? {
-                    $($discr => Self::$var,)*
+                    $($value => Self::$var,)*
                     _ => return None,
                 })
             }
@@ -261,7 +261,7 @@ impl<T: OpArgType> Arg<T> {
     }
     #[inline]
     pub fn new(arg: T) -> (Self, OpArg) {
-        (Self(PhantomData), OpArg(arg.to_oparg()))
+        (Self(PhantomData), OpArg(arg.to_op_arg()))
     }
     #[inline]
     pub fn new_single(arg: T) -> (Self, OpArgByte)
@@ -276,13 +276,13 @@ impl<T: OpArgType> Arg<T> {
     }
     #[inline(always)]
     pub fn try_get(self, arg: OpArg) -> Option<T> {
-        T::from_oparg(arg.0)
+        T::from_op_arg(arg.0)
     }
     #[inline(always)]
     /// # Safety
-    /// T::from_oparg(self) must succeed
+    /// T::from_op_arg(self) must succeed
     pub unsafe fn get_unchecked(self, arg: OpArg) -> T {
-        match T::from_oparg(arg.0) {
+        match T::from_op_arg(arg.0) {
             Some(t) => t,
             None => std::hint::unreachable_unchecked(),
         }
@@ -310,11 +310,11 @@ pub struct Label(pub u32);
 
 impl OpArgType for Label {
     #[inline(always)]
-    fn from_oparg(x: u32) -> Option<Self> {
+    fn from_op_arg(x: u32) -> Option<Self> {
         Some(Label(x))
     }
     #[inline(always)]
-    fn to_oparg(self) -> u32 {
+    fn to_op_arg(self) -> u32 {
         self.0
     }
 }
@@ -325,7 +325,7 @@ impl fmt::Display for Label {
     }
 }
 
-oparg_enum!(
+op_arg_enum!(
     /// Transforms a value prior to formatting it.
     #[derive(Copy, Clone, Debug, PartialEq, Eq)]
     #[repr(u8)]
@@ -344,11 +344,11 @@ oparg_enum!(
 impl TryFrom<usize> for ConversionFlag {
     type Error = usize;
     fn try_from(b: usize) -> Result<Self, Self::Error> {
-        u32::try_from(b).ok().and_then(Self::from_oparg).ok_or(b)
+        u32::try_from(b).ok().and_then(Self::from_op_arg).ok_or(b)
     }
 }
 
-oparg_enum!(
+op_arg_enum!(
     /// The kind of Raise that occurred.
     #[derive(Copy, Clone, Debug, PartialEq, Eq)]
     #[repr(u8)]
@@ -634,11 +634,11 @@ bitflags! {
 }
 impl OpArgType for MakeFunctionFlags {
     #[inline(always)]
-    fn from_oparg(x: u32) -> Option<Self> {
+    fn from_op_arg(x: u32) -> Option<Self> {
         Some(unsafe { MakeFunctionFlags::from_bits_unchecked(x as u8) })
     }
     #[inline(always)]
-    fn to_oparg(self) -> u32 {
+    fn to_op_arg(self) -> u32 {
         self.bits().into()
     }
 }
@@ -793,7 +793,7 @@ impl<C: Constant> BorrowedConstant<'_, C> {
     }
 }
 
-oparg_enum!(
+op_arg_enum!(
     /// The possible comparison operators
     #[derive(Debug, Copy, Clone, PartialEq, Eq)]
     #[repr(u8)]
@@ -809,7 +809,7 @@ oparg_enum!(
     }
 );
 
-oparg_enum!(
+op_arg_enum!(
     #[derive(Debug, Copy, Clone, PartialEq, Eq)]
     #[repr(u8)]
     pub enum TestOperator {
@@ -822,7 +822,7 @@ oparg_enum!(
     }
 );
 
-oparg_enum!(
+op_arg_enum!(
     /// The possible Binary operators
     /// # Examples
     ///
@@ -850,7 +850,7 @@ oparg_enum!(
     }
 );
 
-oparg_enum!(
+op_arg_enum!(
     /// The possible unary operators
     #[derive(Debug, Copy, Clone, PartialEq, Eq)]
     #[repr(u8)]
@@ -870,12 +870,12 @@ pub struct UnpackExArgs {
 
 impl OpArgType for UnpackExArgs {
     #[inline(always)]
-    fn from_oparg(x: u32) -> Option<Self> {
+    fn from_op_arg(x: u32) -> Option<Self> {
         let [before, after, ..] = x.to_le_bytes();
         Some(Self { before, after })
     }
     #[inline(always)]
-    fn to_oparg(self) -> u32 {
+    fn to_op_arg(self) -> u32 {
         u32::from_le_bytes([self.before, self.after, 0, 0])
     }
 }
@@ -925,20 +925,20 @@ impl<C: Constant> CodeObject<C> {
     pub fn arg_names(&self) -> Arguments<C::Name> {
         let nargs = self.arg_count as usize;
         let nkwargs = self.kwonlyarg_count as usize;
-        let mut varargspos = nargs + nkwargs;
+        let mut varargs_pos = nargs + nkwargs;
         let posonlyargs = &self.varnames[..self.posonlyarg_count as usize];
         let args = &self.varnames[..nargs];
-        let kwonlyargs = &self.varnames[nargs..varargspos];
+        let kwonlyargs = &self.varnames[nargs..varargs_pos];
 
         let vararg = if self.flags.contains(CodeFlags::HAS_VARARGS) {
-            let vararg = &self.varnames[varargspos];
-            varargspos += 1;
+            let vararg = &self.varnames[varargs_pos];
+            varargs_pos += 1;
             Some(vararg)
         } else {
             None
         };
         let varkwarg = if self.flags.contains(CodeFlags::HAS_VARKEYWORDS) {
-            Some(&self.varnames[varargspos])
+            Some(&self.varnames[varargs_pos])
         } else {
             None
         };
@@ -968,7 +968,7 @@ impl<C: Constant> CodeObject<C> {
     fn display_inner(
         &self,
         f: &mut fmt::Formatter,
-        expand_codeobjects: bool,
+        expand_code_objects: bool,
         level: usize,
     ) -> fmt::Result {
         let label_targets = self.label_targets();
@@ -1007,14 +1007,14 @@ impl<C: Constant> CodeObject<C> {
             write!(f, "{arrow} {offset:offset_digits$} ")?;
 
             // instruction
-            instruction.fmt_dis(arg, f, self, expand_codeobjects, 21, level)?;
+            instruction.fmt_dis(arg, f, self, expand_code_objects, 21, level)?;
             writeln!(f)?;
         }
         Ok(())
     }
 
     /// Recursively display this CodeObject
-    pub fn display_expand_codeobjects(&self) -> impl fmt::Display + '_ {
+    pub fn display_expand_code_objects(&self) -> impl fmt::Display + '_ {
         struct Display<'a, C: Constant>(&'a CodeObject<C>);
         impl<C: Constant> fmt::Display for Display<'_, C> {
             fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
@@ -1287,7 +1287,7 @@ impl Instruction {
         arg: OpArg,
         f: &mut fmt::Formatter,
         ctx: &impl InstrDisplayContext,
-        expand_codeobjects: bool,
+        expand_code_objects: bool,
         pad: usize,
         level: usize,
     ) -> fmt::Result {
@@ -1295,26 +1295,26 @@ impl Instruction {
             ($variant:ident) => {
                 write!(f, stringify!($variant))
             };
-            ($variant:ident, $map:ident = $argmarker:expr) => {{
-                let arg = $argmarker.get(arg);
+            ($variant:ident, $map:ident = $arg_marker:expr) => {{
+                let arg = $arg_marker.get(arg);
                 write!(f, "{:pad$}({}, {})", stringify!($variant), arg, $map(arg))
             }};
-            ($variant:ident, $argmarker:expr) => {
-                write!(f, "{:pad$}({})", stringify!($variant), $argmarker.get(arg))
+            ($variant:ident, $arg_marker:expr) => {
+                write!(f, "{:pad$}({})", stringify!($variant), $arg_marker.get(arg))
             };
-            ($variant:ident, ?$argmarker:expr) => {
+            ($variant:ident, ?$arg_marker:expr) => {
                 write!(
                     f,
                     "{:pad$}({:?})",
                     stringify!($variant),
-                    $argmarker.get(arg)
+                    $arg_marker.get(arg)
                 )
             };
         }
 
         let varname = |i: u32| ctx.get_varname(i as usize);
         let name = |i: u32| ctx.get_name(i as usize);
-        let cellname = |i: u32| ctx.get_cellname(i as usize);
+        let cell_name = |i: u32| ctx.get_cell_name(i as usize);
 
         match self {
             ImportName { idx } => w!(ImportName, name = idx),
@@ -1324,17 +1324,17 @@ impl Instruction {
             LoadFast(idx) => w!(LoadFast, varname = idx),
             LoadNameAny(idx) => w!(LoadNameAny, name = idx),
             LoadGlobal(idx) => w!(LoadGlobal, name = idx),
-            LoadDeref(idx) => w!(LoadDeref, cellname = idx),
-            LoadClassDeref(idx) => w!(LoadClassDeref, cellname = idx),
+            LoadDeref(idx) => w!(LoadDeref, cell_name = idx),
+            LoadClassDeref(idx) => w!(LoadClassDeref, cell_name = idx),
             StoreFast(idx) => w!(StoreFast, varname = idx),
             StoreLocal(idx) => w!(StoreLocal, name = idx),
             StoreGlobal(idx) => w!(StoreGlobal, name = idx),
-            StoreDeref(idx) => w!(StoreDeref, cellname = idx),
+            StoreDeref(idx) => w!(StoreDeref, cell_name = idx),
             DeleteFast(idx) => w!(DeleteFast, varname = idx),
             DeleteLocal(idx) => w!(DeleteLocal, name = idx),
             DeleteGlobal(idx) => w!(DeleteGlobal, name = idx),
-            DeleteDeref(idx) => w!(DeleteDeref, cellname = idx),
-            LoadClosure(i) => w!(LoadClosure, cellname = i),
+            DeleteDeref(idx) => w!(DeleteDeref, cell_name = idx),
+            LoadClosure(i) => w!(LoadClosure, cell_name = i),
             Subscript => w!(Subscript),
             StoreSubscript => w!(StoreSubscript),
             DeleteSubscript => w!(DeleteSubscript),
@@ -1343,7 +1343,7 @@ impl Instruction {
             LoadConst { idx } => {
                 let value = ctx.get_constant(idx.get(arg) as usize);
                 match value.borrow_constant() {
-                    BorrowedConstant::Code { code } if expand_codeobjects => {
+                    BorrowedConstant::Code { code } if expand_code_objects => {
                         write!(f, "{:pad$}({:?}):", "LoadConst", code)?;
                         code.display_inner(f, true, level + 1)?;
                         Ok(())
@@ -1434,7 +1434,7 @@ pub trait InstrDisplayContext {
     fn get_constant(&self, i: usize) -> &Self::Constant;
     fn get_name(&self, i: usize) -> &str;
     fn get_varname(&self, i: usize) -> &str;
-    fn get_cellname(&self, i: usize) -> &str;
+    fn get_cell_name(&self, i: usize) -> &str;
 }
 
 impl<C: Constant> InstrDisplayContext for CodeObject<C> {
@@ -1448,7 +1448,7 @@ impl<C: Constant> InstrDisplayContext for CodeObject<C> {
     fn get_varname(&self, i: usize) -> &str {
         self.varnames[i].as_ref()
     }
-    fn get_cellname(&self, i: usize) -> &str {
+    fn get_cell_name(&self, i: usize) -> &str {
         self.cellvars
             .get(i)
             .unwrap_or_else(|| &self.freevars[i - self.cellvars.len()])
@@ -1491,7 +1491,7 @@ pub mod frozen_lib {
     }
 
     impl<B: AsRef<[u8]>> FrozenCodeObject<B> {
-        /// Decode a frozen codeobject
+        /// Decode a frozen code object
         #[inline]
         pub fn decode<Bag: AsBag>(
             &self,
diff --git a/core/src/location.rs b/core/src/location.rs
index a6e7d391..33841392 100644
--- a/core/src/location.rs
+++ b/core/src/location.rs
@@ -1,7 +1,7 @@
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
 
-/// Sourcecode location.
+/// Source code location.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct Location {
diff --git a/parser/build.rs b/parser/build.rs
index e693601d..1ef1d537 100644
--- a/parser/build.rs
+++ b/parser/build.rs
@@ -112,7 +112,7 @@ fn gen_phf(out_dir: &Path) {
         .entry("False", "Tok::False")
         .entry("None", "Tok::None")
         .entry("True", "Tok::True")
-        // moreso "standard" keywords
+        // more so "standard" keywords
         .entry("and", "Tok::And")
         .entry("as", "Tok::As")
         .entry("assert", "Tok::Assert")
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 45f5ca8b..38b1d53e 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -241,6 +241,7 @@ where
         lxr.window.slide();
         lxr.window.slide();
         // TODO: Handle possible mismatch between BOM and explicit encoding declaration.
+        // spell-checker:ignore feff
         if let Some('\u{feff}') = lxr.window[0] {
             lxr.window.slide();
         }
@@ -1085,7 +1086,7 @@ where
                 }
             }
             ' ' | '\t' | '\x0C' => {
-                // Skip whitespaces
+                // Skip white-spaces
                 self.next_char();
                 while let Some(' ' | '\t' | '\x0C') = self.window[0] {
                     self.next_char();
@@ -1327,7 +1328,7 @@ mod tests {
         lexer.map(|x| x.unwrap().1).collect()
     }
 
-    fn stok(s: &str) -> Tok {
+    fn str_tok(s: &str) -> Tok {
         Tok::String {
             value: s.to_owned(),
             kind: StringKind::String,
@@ -1335,7 +1336,7 @@ mod tests {
         }
     }
 
-    fn raw_stok(s: &str) -> Tok {
+    fn raw_str_tok(s: &str) -> Tok {
         Tok::String {
             value: s.to_owned(),
             kind: StringKind::RawString,
@@ -1434,13 +1435,13 @@ mod tests {
 
     #[test]
     fn test_assignment() {
-        let source = r"avariable = 99 + 2-0";
+        let source = r"a_variable = 99 + 2-0";
         let tokens = lex_source(source);
         assert_eq!(
             tokens,
             vec![
                 Tok::Name {
-                    name: String::from("avariable"),
+                    name: String::from("a_variable"),
                 },
                 Tok::Equal,
                 Tok::Int {
@@ -1663,13 +1664,13 @@ mod tests {
             vec![
                 Tok::Lpar,
                 Tok::NonLogicalNewline,
-                stok("a"),
+                str_tok("a"),
                 Tok::NonLogicalNewline,
-                stok("b"),
+                str_tok("b"),
                 Tok::NonLogicalNewline,
                 Tok::NonLogicalNewline,
-                stok("c"),
-                stok("d"),
+                str_tok("c"),
+                str_tok("d"),
                 Tok::NonLogicalNewline,
                 Tok::Rpar,
                 Tok::Newline,
@@ -1716,15 +1717,15 @@ mod tests {
         assert_eq!(
             tokens,
             vec![
-                stok("double"),
-                stok("single"),
-                stok(r"can\'t"),
-                stok(r#"\\\""#),
-                stok(r"\t\r\n"),
-                stok(r"\g"),
-                raw_stok(r"raw\'"),
-                stok(r"\420"),
-                stok(r"\200\0a"),
+                str_tok("double"),
+                str_tok("single"),
+                str_tok(r"can\'t"),
+                str_tok(r#"\\\""#),
+                str_tok(r"\t\r\n"),
+                str_tok(r"\g"),
+                raw_str_tok(r"raw\'"),
+                str_tok(r"\420"),
+                str_tok(r"\200\0a"),
                 Tok::Newline,
             ]
         );
@@ -1740,7 +1741,7 @@ mod tests {
                 assert_eq!(
                     tokens,
                     vec![
-                        stok("abc\\\ndef"),
+                        str_tok("abc\\\ndef"),
                         Tok::Newline,
                     ]
                 )
@@ -1759,7 +1760,7 @@ mod tests {
     fn test_escape_unicode_name() {
         let source = r#""\N{EN SPACE}""#;
         let tokens = lex_source(source);
-        assert_eq!(tokens, vec![stok(r"\N{EN SPACE}"), Tok::Newline])
+        assert_eq!(tokens, vec![str_tok(r"\N{EN SPACE}"), Tok::Newline])
     }
 
     macro_rules! test_triple_quoted {
diff --git a/parser/src/parser.rs b/parser/src/parser.rs
index 6376e046..b9c0bc0a 100644
--- a/parser/src/parser.rs
+++ b/parser/src/parser.rs
@@ -433,14 +433,14 @@ class Foo(A, B):
     }
 
     #[test]
-    fn test_parse_boolop_or() {
+    fn test_parse_bool_op_or() {
         let source = "x or y";
         let parse_ast = parse_expression(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
-    fn test_parse_boolop_and() {
+    fn test_parse_bool_op_and() {
         let source = "x and y";
         let parse_ast = parse_expression(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
@@ -513,10 +513,10 @@ with (0 as a, 1 as b,): pass
     #[test]
     fn test_star_index() {
         let source = "\
-array_slice = array[0, *idxs, -1]
-array[0, *idxs, -1] = array_slice
-array[*idxs_to_select, *idxs_to_select]
-array[3:5, *idxs_to_select]
+array_slice = array[0, *indexes, -1]
+array[0, *indexes, -1] = array_slice
+array[*indexes_to_select, *indexes_to_select]
+array[3:5, *indexes_to_select]
 ";
         let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
diff --git a/parser/src/snapshots/rustpython_parser__context__tests__assign.snap b/parser/src/snapshots/rustpython_parser__context__tests__assign.snap
deleted file mode 100644
index a9fe441d..00000000
--- a/parser/src/snapshots/rustpython_parser__context__tests__assign.snap
+++ /dev/null
@@ -1,42 +0,0 @@
----
-source: compiler/parser/src/context.rs
-expression: parse_ast
----
-[
-    Located {
-        location: Location {
-            row: 1,
-            column: 1,
-        },
-        custom: (),
-        node: Assign {
-            targets: [
-                Located {
-                    location: Location {
-                        row: 1,
-                        column: 1,
-                    },
-                    custom: (),
-                    node: Name {
-                        id: "x",
-                        ctx: Store,
-                    },
-                },
-            ],
-            value: Located {
-                location: Location {
-                    row: 1,
-                    column: 5,
-                },
-                custom: (),
-                node: Constant {
-                    value: Int(
-                        1,
-                    ),
-                    kind: None,
-                },
-            },
-            type_comment: None,
-        },
-    },
-]
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args.snap b/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args.snap
index bfb7074e..2b990fa8 100644
--- a/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args.snap
+++ b/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/function.rs
-assertion_line: 165
 expression: parse_ast
 ---
 Ok(
diff --git a/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args_with_defaults.snap b/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args_with_defaults.snap
index 2004bbee..08fa27e4 100644
--- a/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args_with_defaults.snap
+++ b/parser/src/snapshots/rustpython_parser__function__tests__function_kw_only_args_with_defaults.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/function.rs
-assertion_line: 165
 expression: parse_ast
 ---
 Ok(
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_boolop_and.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_bool_op_and.snap
similarity index 100%
rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_boolop_and.snap
rename to parser/src/snapshots/rustpython_parser__parser__tests__parse_bool_op_and.snap
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_boolop_or.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_bool_op_or.snap
similarity index 100%
rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_boolop_or.snap
rename to parser/src/snapshots/rustpython_parser__parser__tests__parse_bool_op_or.snap
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap b/parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap
index 64d89c15..17be8fe7 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap
@@ -1,5 +1,5 @@
 ---
-source: parser/src/parser.rs
+source: compiler/parser/src/parser.rs
 expression: parse_ast
 ---
 []
diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__star_index.snap b/parser/src/snapshots/rustpython_parser__parser__tests__star_index.snap
index 6aed23b8..897edb23 100644
--- a/parser/src/snapshots/rustpython_parser__parser__tests__star_index.snap
+++ b/parser/src/snapshots/rustpython_parser__parser__tests__star_index.snap
@@ -11,7 +11,7 @@ expression: parse_ast
         end_location: Some(
             Location {
                 row: 1,
-                column: 33,
+                column: 36,
             },
         ),
         custom: (),
@@ -43,7 +43,7 @@ expression: parse_ast
                 end_location: Some(
                     Location {
                         row: 1,
-                        column: 33,
+                        column: 36,
                     },
                 ),
                 custom: (),
@@ -73,7 +73,7 @@ expression: parse_ast
                         end_location: Some(
                             Location {
                                 row: 1,
-                                column: 32,
+                                column: 35,
                             },
                         ),
                         custom: (),
@@ -106,7 +106,7 @@ expression: parse_ast
                                     end_location: Some(
                                         Location {
                                             row: 1,
-                                            column: 28,
+                                            column: 31,
                                         },
                                     ),
                                     custom: (),
@@ -119,12 +119,12 @@ expression: parse_ast
                                             end_location: Some(
                                                 Location {
                                                     row: 1,
-                                                    column: 28,
+                                                    column: 31,
                                                 },
                                             ),
                                             custom: (),
                                             node: Name {
-                                                id: "idxs",
+                                                id: "indexes",
                                                 ctx: Load,
                                             },
                                         },
@@ -134,12 +134,12 @@ expression: parse_ast
                                 Located {
                                     location: Location {
                                         row: 1,
-                                        column: 30,
+                                        column: 33,
                                     },
                                     end_location: Some(
                                         Location {
                                             row: 1,
-                                            column: 32,
+                                            column: 35,
                                         },
                                     ),
                                     custom: (),
@@ -148,12 +148,12 @@ expression: parse_ast
                                         operand: Located {
                                             location: Location {
                                                 row: 1,
-                                                column: 31,
+                                                column: 34,
                                             },
                                             end_location: Some(
                                                 Location {
                                                     row: 1,
-                                                    column: 32,
+                                                    column: 35,
                                                 },
                                             ),
                                             custom: (),
@@ -184,7 +184,7 @@ expression: parse_ast
         end_location: Some(
             Location {
                 row: 2,
-                column: 33,
+                column: 36,
             },
         ),
         custom: (),
@@ -198,7 +198,7 @@ expression: parse_ast
                     end_location: Some(
                         Location {
                             row: 2,
-                            column: 19,
+                            column: 22,
                         },
                     ),
                     custom: (),
@@ -228,7 +228,7 @@ expression: parse_ast
                             end_location: Some(
                                 Location {
                                     row: 2,
-                                    column: 18,
+                                    column: 21,
                                 },
                             ),
                             custom: (),
@@ -261,7 +261,7 @@ expression: parse_ast
                                         end_location: Some(
                                             Location {
                                                 row: 2,
-                                                column: 14,
+                                                column: 17,
                                             },
                                         ),
                                         custom: (),
@@ -274,12 +274,12 @@ expression: parse_ast
                                                 end_location: Some(
                                                     Location {
                                                         row: 2,
-                                                        column: 14,
+                                                        column: 17,
                                                     },
                                                 ),
                                                 custom: (),
                                                 node: Name {
-                                                    id: "idxs",
+                                                    id: "indexes",
                                                     ctx: Load,
                                                 },
                                             },
@@ -289,12 +289,12 @@ expression: parse_ast
                                     Located {
                                         location: Location {
                                             row: 2,
-                                            column: 16,
+                                            column: 19,
                                         },
                                         end_location: Some(
                                             Location {
                                                 row: 2,
-                                                column: 18,
+                                                column: 21,
                                             },
                                         ),
                                         custom: (),
@@ -303,12 +303,12 @@ expression: parse_ast
                                             operand: Located {
                                                 location: Location {
                                                     row: 2,
-                                                    column: 17,
+                                                    column: 20,
                                                 },
                                                 end_location: Some(
                                                     Location {
                                                         row: 2,
-                                                        column: 18,
+                                                        column: 21,
                                                     },
                                                 ),
                                                 custom: (),
@@ -332,12 +332,12 @@ expression: parse_ast
             value: Located {
                 location: Location {
                     row: 2,
-                    column: 22,
+                    column: 25,
                 },
                 end_location: Some(
                     Location {
                         row: 2,
-                        column: 33,
+                        column: 36,
                     },
                 ),
                 custom: (),
@@ -357,7 +357,7 @@ expression: parse_ast
         end_location: Some(
             Location {
                 row: 3,
-                column: 39,
+                column: 45,
             },
         ),
         custom: (),
@@ -370,7 +370,7 @@ expression: parse_ast
                 end_location: Some(
                     Location {
                         row: 3,
-                        column: 39,
+                        column: 45,
                     },
                 ),
                 custom: (),
@@ -400,7 +400,7 @@ expression: parse_ast
                         end_location: Some(
                             Location {
                                 row: 3,
-                                column: 38,
+                                column: 44,
                             },
                         ),
                         custom: (),
@@ -414,7 +414,7 @@ expression: parse_ast
                                     end_location: Some(
                                         Location {
                                             row: 3,
-                                            column: 21,
+                                            column: 24,
                                         },
                                     ),
                                     custom: (),
@@ -427,12 +427,12 @@ expression: parse_ast
                                             end_location: Some(
                                                 Location {
                                                     row: 3,
-                                                    column: 21,
+                                                    column: 24,
                                                 },
                                             ),
                                             custom: (),
                                             node: Name {
-                                                id: "idxs_to_select",
+                                                id: "indexes_to_select",
                                                 ctx: Load,
                                             },
                                         },
@@ -442,12 +442,12 @@ expression: parse_ast
                                 Located {
                                     location: Location {
                                         row: 3,
-                                        column: 23,
+                                        column: 26,
                                     },
                                     end_location: Some(
                                         Location {
                                             row: 3,
-                                            column: 38,
+                                            column: 44,
                                         },
                                     ),
                                     custom: (),
@@ -455,17 +455,17 @@ expression: parse_ast
                                         value: Located {
                                             location: Location {
                                                 row: 3,
-                                                column: 24,
+                                                column: 27,
                                             },
                                             end_location: Some(
                                                 Location {
                                                     row: 3,
-                                                    column: 38,
+                                                    column: 44,
                                                 },
                                             ),
                                             custom: (),
                                             node: Name {
-                                                id: "idxs_to_select",
+                                                id: "indexes_to_select",
                                                 ctx: Load,
                                             },
                                         },
@@ -489,7 +489,7 @@ expression: parse_ast
         end_location: Some(
             Location {
                 row: 4,
-                column: 27,
+                column: 30,
             },
         ),
         custom: (),
@@ -502,7 +502,7 @@ expression: parse_ast
                 end_location: Some(
                     Location {
                         row: 4,
-                        column: 27,
+                        column: 30,
                     },
                 ),
                 custom: (),
@@ -532,7 +532,7 @@ expression: parse_ast
                         end_location: Some(
                             Location {
                                 row: 4,
-                                column: 26,
+                                column: 29,
                             },
                         ),
                         custom: (),
@@ -604,7 +604,7 @@ expression: parse_ast
                                     end_location: Some(
                                         Location {
                                             row: 4,
-                                            column: 26,
+                                            column: 29,
                                         },
                                     ),
                                     custom: (),
@@ -617,12 +617,12 @@ expression: parse_ast
                                             end_location: Some(
                                                 Location {
                                                     row: 4,
-                                                    column: 26,
+                                                    column: 29,
                                                 },
                                             ),
                                             custom: (),
                                             node: Name {
-                                                id: "idxs_to_select",
+                                                id: "indexes_to_select",
                                                 ctx: Load,
                                             },
                                         },
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base.snap b/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_self_documenting_base.snap
similarity index 98%
rename from parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_self_documenting_base.snap
index cca26adc..125c6af7 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_self_documenting_base.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/string.rs
-assertion_line: 698
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base_more.snap b/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_self_documenting_base_more.snap
similarity index 99%
rename from parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base_more.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_self_documenting_base_more.snap
index c425d676..14a20345 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_base_more.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_self_documenting_base_more.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/string.rs
-assertion_line: 706
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_format.snap b/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_self_documenting_format.snap
similarity index 99%
rename from parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_format.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_self_documenting_format.snap
index c402ed68..c7f62883 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_selfdocumenting_format.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__fstring_parse_self_documenting_format.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/string.rs
-assertion_line: 714
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_empty_fstring.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_empty_fstring.snap
index 073ac428..94515b97 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_empty_fstring.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_empty_fstring.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/string.rs
-assertion_line: 690
 expression: "parse_fstring(\"\").unwrap()"
 ---
 []
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring.snap
index 10b6f3ac..4355fd70 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/string.rs
-assertion_line: 669
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_equals.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_equals.snap
index 194061c3..c8c20522 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_equals.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/string.rs
-assertion_line: 766
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_nested_spec.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_nested_spec.snap
index 96ec3e72..61549489 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_nested_spec.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/string.rs
-assertion_line: 677
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_equals.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_equals.snap
index cbfe435d..029b5ee1 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_equals.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_equals.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/string.rs
-assertion_line: 759
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_nested_spec.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_nested_spec.snap
index 2adc5a48..8b26d295 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_nested_spec.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_not_nested_spec.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/string.rs
-assertion_line: 685
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_prec_space.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_self_doc_prec_space.snap
similarity index 98%
rename from parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_prec_space.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_self_doc_prec_space.snap
index 16d62ea1..f48f688e 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_prec_space.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_self_doc_prec_space.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/string.rs
-assertion_line: 773
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_trailing_space.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_self_doc_trailing_space.snap
similarity index 98%
rename from parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_trailing_space.snap
rename to parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_self_doc_trailing_space.snap
index 16171f2f..e1ced214 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_selfdoc_trailing_space.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_self_doc_trailing_space.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/string.rs
-assertion_line: 780
 expression: parse_ast
 ---
 [
diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_yield_expr.snap b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_yield_expr.snap
index e3a9ff79..90223888 100644
--- a/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_yield_expr.snap
+++ b/parser/src/snapshots/rustpython_parser__string__tests__parse_fstring_yield_expr.snap
@@ -1,6 +1,5 @@
 ---
 source: compiler/parser/src/string.rs
-assertion_line: 787
 expression: parse_ast
 ---
 [
diff --git a/parser/src/soft_keywords.rs b/parser/src/soft_keywords.rs
index b7b034f5..c670fd17 100644
--- a/parser/src/soft_keywords.rs
+++ b/parser/src/soft_keywords.rs
@@ -27,7 +27,7 @@ where
 {
     pub fn new(lexer: I, mode: Mode) -> Self {
         Self {
-            underlying: lexer.multipeek(),
+            underlying: lexer.multipeek(), // spell-checker:ignore multipeek
             start_of_line: matches!(mode, Mode::Interactive | Mode::Module),
         }
     }
diff --git a/parser/src/string.rs b/parser/src/string.rs
index 0cd03df6..14f5ce1c 100644
--- a/parser/src/string.rs
+++ b/parser/src/string.rs
@@ -179,7 +179,7 @@ impl<'a> StringParser<'a> {
 
         let mut expression = String::new();
         let mut spec = None;
-        let mut delims = Vec::new();
+        let mut delimiters = Vec::new();
         let mut conversion = ConversionFlag::None;
         let mut self_documenting = false;
         let mut trailing_seq = String::new();
@@ -194,7 +194,7 @@ impl<'a> StringParser<'a> {
                     expression.push('=');
                     self.next_char();
                 }
-                '!' if delims.is_empty() && self.peek() != Some(&'=') => {
+                '!' if delimiters.is_empty() && self.peek() != Some(&'=') => {
                     if expression.trim().is_empty() {
                         return Err(FStringError::new(EmptyExpression, self.get_pos()).into());
                     }
@@ -223,11 +223,11 @@ impl<'a> StringParser<'a> {
 
                 // match a python 3.8 self documenting expression
                 // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}'
-                '=' if self.peek() != Some(&'=') && delims.is_empty() => {
+                '=' if self.peek() != Some(&'=') && delimiters.is_empty() => {
                     self_documenting = true;
                 }
 
-                ':' if delims.is_empty() => {
+                ':' if delimiters.is_empty() => {
                     let parsed_spec = self.parse_spec(nested)?;
 
                     spec = Some(Box::new(self.expr(ExprKind::JoinedStr {
@@ -236,10 +236,10 @@ impl<'a> StringParser<'a> {
                 }
                 '(' | '{' | '[' => {
                     expression.push(ch);
-                    delims.push(ch);
+                    delimiters.push(ch);
                 }
                 ')' => {
-                    let last_delim = delims.pop();
+                    let last_delim = delimiters.pop();
                     match last_delim {
                         Some('(') => {
                             expression.push(ch);
@@ -257,7 +257,7 @@ impl<'a> StringParser<'a> {
                     }
                 }
                 ']' => {
-                    let last_delim = delims.pop();
+                    let last_delim = delimiters.pop();
                     match last_delim {
                         Some('[') => {
                             expression.push(ch);
@@ -274,8 +274,8 @@ impl<'a> StringParser<'a> {
                         }
                     }
                 }
-                '}' if !delims.is_empty() => {
-                    let last_delim = delims.pop();
+                '}' if !delimiters.is_empty() => {
+                    let last_delim = delimiters.pop();
                     match last_delim {
                         Some('{') => {
                             expression.push(ch);
@@ -800,7 +800,7 @@ mod tests {
     }
 
     #[test]
-    fn test_fstring_parse_selfdocumenting_base() {
+    fn test_fstring_parse_self_documenting_base() {
         let src = "{user=}";
         let parse_ast = parse_fstring(src).unwrap();
 
@@ -808,7 +808,7 @@ mod tests {
     }
 
     #[test]
-    fn test_fstring_parse_selfdocumenting_base_more() {
+    fn test_fstring_parse_self_documenting_base_more() {
         let src = "mix {user=} with text and {second=}";
         let parse_ast = parse_fstring(src).unwrap();
 
@@ -816,7 +816,7 @@ mod tests {
     }
 
     #[test]
-    fn test_fstring_parse_selfdocumenting_format() {
+    fn test_fstring_parse_self_documenting_format() {
         let src = "{user=:>10}";
         let parse_ast = parse_fstring(src).unwrap();
 
@@ -877,14 +877,14 @@ mod tests {
     }
 
     #[test]
-    fn test_parse_fstring_selfdoc_prec_space() {
+    fn test_parse_fstring_self_doc_prec_space() {
         let source = "{x   =}";
         let parse_ast = parse_fstring(source).unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
-    fn test_parse_fstring_selfdoc_trailing_space() {
+    fn test_parse_fstring_self_doc_trailing_space() {
         let source = "{x=   }";
         let parse_ast = parse_fstring(source).unwrap();
         insta::assert_debug_snapshot!(parse_ast);
@@ -979,7 +979,7 @@ mod tests {
     #[test]
     fn test_escape_char_in_byte_literal() {
         // backslash does not escape
-        let source = r##"b"omkmok\Xaa""##;
+        let source = r##"b"omkmok\Xaa""##; // spell-checker:ignore omkmok
         let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
diff --git a/src/lib.rs b/src/lib.rs
index 27c15872..9c38408b 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -50,7 +50,7 @@ fn error_from_parse(error: parser::ParseError, source: &str) -> CompileError {
     CompileError::from(error, source)
 }
 
-/// Compile a given sourcecode into a bytecode object.
+/// Compile a given source code into a bytecode object.
 pub fn compile(
     source: &str,
     mode: compile::Mode,

From 9155ed38263879011f5574edf350544a4350b309 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Sat, 18 Mar 2023 21:50:08 +0900
Subject: [PATCH 451/459] Remove useless and_then

---
 codegen/src/symboltable.rs | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/codegen/src/symboltable.rs b/codegen/src/symboltable.rs
index 258d3577..b53e8844 100644
--- a/codegen/src/symboltable.rs
+++ b/codegen/src/symboltable.rs
@@ -414,21 +414,20 @@ impl SymbolTableAnalyzer {
         st_typ: SymbolTableType,
     ) -> Option<SymbolScope> {
         sub_tables.iter().find_map(|st| {
-            st.symbols.get(name).and_then(|sym| {
-                if sym.scope == SymbolScope::Free || sym.flags.contains(SymbolFlags::FREE_CLASS) {
-                    if st_typ == SymbolTableType::Class && name != "__class__" {
-                        None
-                    } else {
-                        Some(SymbolScope::Cell)
-                    }
-                } else if sym.scope == SymbolScope::GlobalExplicit && self.tables.is_empty() {
-                    // the symbol is defined on the module level, and an inner scope declares
-                    // a global that points to it
-                    Some(SymbolScope::GlobalExplicit)
-                } else {
+            let sym = st.symbols.get(name)?;
+            if sym.scope == SymbolScope::Free || sym.flags.contains(SymbolFlags::FREE_CLASS) {
+                if st_typ == SymbolTableType::Class && name != "__class__" {
                     None
+                } else {
+                    Some(SymbolScope::Cell)
                 }
-            })
+            } else if sym.scope == SymbolScope::GlobalExplicit && self.tables.is_empty() {
+                // the symbol is defined on the module level, and an inner scope declares
+                // a global that points to it
+                Some(SymbolScope::GlobalExplicit)
+            } else {
+                None
+            }
         })
     }
 

From ce3cc2831e2079d3d7fa9a2ee9188ea38b881df3 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Sat, 25 Mar 2023 22:18:02 +0900
Subject: [PATCH 452/459] Update lalrpop

---
 parser/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parser/Cargo.toml b/parser/Cargo.toml
index 1cb80746..0568fbb2 100644
--- a/parser/Cargo.toml
+++ b/parser/Cargo.toml
@@ -14,7 +14,7 @@ serde = ["dep:serde", "rustpython-compiler-core/serde"]
 
 [build-dependencies]
 anyhow = { workspace = true }
-lalrpop = { version = "0.19.8", optional = true }
+lalrpop = { version = "0.19.9", optional = true }
 phf_codegen = "0.11.1"
 tiny-keccak = { version = "2", features = ["sha3"] }
 

From 457cf8ba44e9b189dde2866aab0abcf1ddcaf11c Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Thu, 20 Apr 2023 17:33:17 +0900
Subject: [PATCH 453/459] module objects' type as PyModule

---
 ast/asdl_rs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py
index b217e629..8caeca90 100755
--- a/ast/asdl_rs.py
+++ b/ast/asdl_rs.py
@@ -474,7 +474,7 @@ class ExtendModuleVisitor(EmitVisitor):
     def visitModule(self, mod):
         depth = 0
         self.emit(
-            "pub fn extend_module_nodes(vm: &VirtualMachine, module: &PyObject) {",
+            "pub fn extend_module_nodes(vm: &VirtualMachine, module: &Py<PyModule>) {",
             depth,
         )
         self.emit("extend_module!(vm, module, {", depth + 1)

From 321ab8382507e90341004ac810b26f4e986245e7 Mon Sep 17 00:00:00 2001
From: Micha Reiser <micha@reiser.io>
Date: Tue, 25 Apr 2023 17:49:37 -0600
Subject: [PATCH 454/459] Use `Located::new` over struct initializer

---
 parser/python.lalrpop | 732 +++++++++++++++++++-----------------------
 1 file changed, 326 insertions(+), 406 deletions(-)

diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index c8936d33..32c08809 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -67,23 +67,21 @@ SmallStatement: ast::Stmt = {
 
 PassStatement: ast::Stmt = {
     <location:@L> "pass" <end_location:@R> => {
-        ast::Stmt {
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::StmtKind::Pass,
-        }
+            end_location,
+            ast::StmtKind::Pass,
+        )
     },
 };
 
 DelStatement: ast::Stmt = {
     <location:@L> "del" <targets:ExpressionList2> <end_location:@R> => {
-        ast::Stmt {
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::StmtKind::Delete { targets: targets.into_iter().map(|expr| set_context(expr, ast::ExprContext::Del)).collect() },
-        }
+            end_location,
+            ast::StmtKind::Delete { targets: targets.into_iter().map(|expr| set_context(expr, ast::ExprContext::Del)).collect() },
+        )
     },
 };
 
@@ -91,12 +89,11 @@ ExpressionStatement: ast::Stmt = {
     <location:@L> <expression:TestOrStarExprList> <suffix:AssignSuffix*> <end_location:@R> => {
         // Just an expression, no assignment:
         if suffix.is_empty() {
-            ast::Stmt {
-                custom: (),
+            ast::Stmt::new(
                 location,
-                end_location: Some(end_location),
-                node: ast::StmtKind::Expr { value: Box::new(expression) }
-            }
+                end_location,
+                ast::StmtKind::Expr { value: Box::new(expression) }
+            )
         } else {
             let mut targets = vec![set_context(expression, ast::ExprContext::Store)];
             let mut values = suffix;
@@ -107,39 +104,36 @@ ExpressionStatement: ast::Stmt = {
 
             let value = Box::new(values.into_iter().next().unwrap());
 
-            ast::Stmt {
-                custom: (),
+            ast::Stmt::new(
                 location,
-                end_location: Some(end_location),
-                node: ast::StmtKind::Assign { targets, value, type_comment: None },
-            }
+                end_location,
+                ast::StmtKind::Assign { targets, value, type_comment: None },
+            )
         }
     },
     <location:@L> <target:TestOrStarExprList> <op:AugAssign> <rhs:TestListOrYieldExpr> <end_location:@R> => {
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::AugAssign {
+            end_location,
+            ast::StmtKind::AugAssign {
                 target: Box::new(set_context(target, ast::ExprContext::Store)),
                 op,
                 value: Box::new(rhs)
             },
-        }
+        )
     },
     <location:@L> <target:Test<"all">> ":" <annotation:Test<"all">> <rhs:AssignSuffix?> <end_location:@R> => {
         let simple = matches!(target.node, ast::ExprKind::Name { .. });
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::AnnAssign {
+            end_location,
+            ast::StmtKind::AnnAssign {
                 target: Box::new(set_context(target, ast::ExprContext::Store)),
                 annotation: Box::new(annotation),
                 value: rhs.map(Box::new),
                 simple: if simple { 1 } else { 0 },
             },
-        }
+        )
     },
 };
 
@@ -191,80 +185,72 @@ AugAssign: ast::Operator = {
 
 FlowStatement: ast::Stmt = {
     <location:@L> "break" <end_location:@R> => {
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::Break,
-        }
+            end_location,
+            ast::StmtKind::Break,
+        )
     },
     <location:@L> "continue" <end_location:@R> => {
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::Continue,
-        }
+            end_location,
+            ast::StmtKind::Continue,
+        )
     },
     <location:@L> "return" <value:TestList?> <end_location:@R> => {
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::Return { value: value.map(Box::new) },
-        }
+            end_location,
+            ast::StmtKind::Return { value: value.map(Box::new) },
+        )
     },
     <location:@L> <expression:YieldExpr> <end_location:@R> => {
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::Expr { value: Box::new(expression) },
-        }
+            end_location,
+            ast::StmtKind::Expr { value: Box::new(expression) },
+        )
     },
     RaiseStatement,
 };
 
 RaiseStatement: ast::Stmt = {
     <location:@L> "raise" <end_location:@R> => {
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::Raise { exc: None, cause: None },
-        }
+            end_location,
+            ast::StmtKind::Raise { exc: None, cause: None },
+        )
     },
     <location:@L> "raise" <t:Test<"all">> <c:("from" Test<"all">)?> <end_location:@R> => {
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::Raise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)) },
-        }
+            end_location,
+            ast::StmtKind::Raise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)) },
+        )
     },
 };
 
 ImportStatement: ast::Stmt = {
     <location:@L> "import" <names: OneOrMore<ImportAsAlias<DottedName>>> <end_location:@R> => {
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::Import { names },
-        }
+            end_location,
+            ast::StmtKind::Import { names },
+        )
     },
     <location:@L> "from" <source:ImportFromLocation> "import" <names: ImportAsNames> <end_location:@R> => {
         let (level, module) = source;
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::ImportFrom {
+            end_location,
+            ast::StmtKind::ImportFrom {
                 level,
                 module,
                 names
             },
-        }
+        )
     },
 };
 
@@ -312,37 +298,34 @@ DottedName: String = {
 
 GlobalStatement: ast::Stmt = {
     <location:@L> "global" <names:OneOrMore<Identifier>> <end_location:@R> => {
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::Global { names }
-        }
+            end_location,
+            ast::StmtKind::Global { names }
+        )
     },
 };
 
 NonlocalStatement: ast::Stmt = {
     <location:@L> "nonlocal" <names:OneOrMore<Identifier>> <end_location:@R> => {
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::Nonlocal { names }
-        }
+            end_location,
+            ast::StmtKind::Nonlocal { names }
+        )
     },
 };
 
 AssertStatement: ast::Stmt = {
     <location:@L> "assert" <test:Test<"all">> <msg: ("," Test<"all">)?> <end_location:@R> => {
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::Assert {
+            end_location,
+            ast::StmtKind::Assert {
                 test: Box::new(test),
                 msg: msg.map(|e| Box::new(e.1))
             }
-        }
+        )
     },
 };
 
@@ -367,15 +350,14 @@ MatchStatement: ast::Stmt = {
             .unwrap()
             .end_location
             .unwrap();
-        ast::Stmt {
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::StmtKind::Match {
+            end_location,
+            ast::StmtKind::Match {
                 subject: Box::new(subject),
                 cases
             }
-        }
+        )
     },
     <location:@L> "match" <subject:TestOrStarNamedExpr> "," ":" "\n" Indent <cases:MatchCase+> Dedent => {
        let end_location = cases
@@ -386,15 +368,14 @@ MatchStatement: ast::Stmt = {
             .unwrap()
             .end_location
             .unwrap();
-        ast::Stmt {
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::StmtKind::Match {
+            end_location,
+            ast::StmtKind::Match {
                 subject: Box::new(subject),
                 cases
             }
-        }
+        )
     },
     <location:@L> "match" <subject:TestOrStarNamedExpr> "," <subjects:OneOrMore<TestOrStarNamedExpr>> ","? ":" "\n" Indent <cases:MatchCase+> Dedent => {
         let end_location = cases
@@ -407,23 +388,21 @@ MatchStatement: ast::Stmt = {
             .unwrap();
         let mut subjects = subjects;
         subjects.insert(0, subject);
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
-            end_location: Some(end_location),
-            node: ast::StmtKind::Match {
-                subject: Box::new(ast::Expr {
+            end_location,
+            ast::StmtKind::Match {
+                subject: Box::new(ast::Expr::new(
                     location,
-                    end_location: Some(end_location),
-                    custom: (),
-                    node: ast::ExprKind::Tuple {
+                    end_location,
+                    ast::ExprKind::Tuple {
                         elts: subjects,
                         ctx: ast::ExprContext::Load,
                     },
-                }),
+                )),
                 cases
             }
-        }
+        )
     }
 }
 
@@ -444,25 +423,23 @@ Guard: ast::Expr = {
 }
 
 Patterns: ast::Pattern = {
-    <location:@L> <pattern:Pattern> "," <end_location:@R> => ast::Pattern {
+    <location:@L> <pattern:Pattern> "," <end_location:@R> => ast::Pattern::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::PatternKind::MatchSequence {
+        end_location,
+        ast::PatternKind::MatchSequence {
             patterns: vec![pattern]
         },
-    },
+    ),
     <location:@L> <pattern:Pattern> "," <patterns:OneOrMore<Pattern>> ","? <end_location:@R> => {
         let mut patterns = patterns;
         patterns.insert(0, pattern);
-        ast::Pattern {
+        ast::Pattern::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::PatternKind::MatchSequence {
+            end_location,
+            ast::PatternKind::MatchSequence {
                 patterns
             },
-        }
+        )
     },
     <pattern:Pattern> => pattern
 }
@@ -480,15 +457,14 @@ AsPattern: ast::Pattern = {
                 location,
             })?
         } else {
-            Ok(ast::Pattern {
+            Ok(ast::Pattern::new(
                 location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::PatternKind::MatchAs {
+                end_location,
+                ast::PatternKind::MatchAs {
                    pattern: Some(Box::new(pattern)),
                    name: Some(name),
                },
-            })
+            ))
         }
     },
 }
@@ -498,58 +474,50 @@ OrPattern: ast::Pattern = {
     <location:@L> <pattern:ClosedPattern> <patterns:("|" <ClosedPattern>)+> <end_location:@R> => {
         let mut patterns = patterns;
         patterns.insert(0, pattern);
-        ast::Pattern {
+        ast::Pattern::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::PatternKind::MatchOr { patterns }
-        }
+            end_location,
+            ast::PatternKind::MatchOr { patterns }
+        )
     }
 }
 
 ClosedPattern: ast::Pattern = {
-    <location:@L> <node:LiteralPattern> <end_location:@R> => ast::Pattern {
+    <location:@L> <node:LiteralPattern> <end_location:@R> => ast::Pattern::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
+        end_location,
         node,
-    },
-    <location:@L> <node:CapturePattern> <end_location:@R> => ast::Pattern {
+    ),
+    <location:@L> <node:CapturePattern> <end_location:@R> => ast::Pattern::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
+        end_location,
         node,
-    },
-    <location:@L> <node:StarPattern> <end_location:@R> => ast::Pattern {
+    ),
+    <location:@L> <node:StarPattern> <end_location:@R> => ast::Pattern::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
+        end_location,
         node,
-    },
-    <location:@L> <node:ValuePattern> <end_location:@R> => ast::Pattern {
+    ),
+    <location:@L> <node:ValuePattern> <end_location:@R> => ast::Pattern::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
+        end_location,
         node,
-    },
-    <location:@L> <node:SequencePattern> <end_location:@R> => ast::Pattern {
+    ),
+    <location:@L> <node:SequencePattern> <end_location:@R> => ast::Pattern::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
+        end_location,
         node,
-    },
-    <location:@L> <node:MappingPattern> <end_location:@R> => ast::Pattern {
+    ),
+    <location:@L> <node:MappingPattern> <end_location:@R> => ast::Pattern::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
+        end_location,
         node,
-    },
-    <location:@L> <node:ClassPattern> <end_location:@R> => ast::Pattern {
+    ),
+    <location:@L> <node:ClassPattern> <end_location:@R> => ast::Pattern::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
+        end_location,
         node,
-    },
+    ),
 }
 
 SequencePattern: ast::PatternKind = {
@@ -577,38 +545,35 @@ StarPattern: ast::PatternKind = {
 }
 
 ConstantAtom: ast::Expr = {
-    <location:@L> <value:Constant> <end_location:@R> => ast::Expr {
+    <location:@L> <value:Constant> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Constant { value, kind: None }
-    },
+        end_location,
+        ast::ExprKind::Constant { value, kind: None }
+    ),
 }
 
 ConstantExpr: ast::Expr = {
     ConstantAtom,
-    <location:@L> "-" <operand:ConstantAtom> <end_location:@R> => ast::Expr {
+    <location:@L> "-" <operand:ConstantAtom> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::UnaryOp {
+        end_location,
+        ast::ExprKind::UnaryOp {
             op: ast::Unaryop::USub,
             operand: Box::new(operand)
         }
-    },
+    ),
 }
 
 AddOpExpr: ast::Expr = {
-    <location:@L> <left:ConstantExpr> <op:AddOp> <right:ConstantAtom> <end_location:@R> => ast::Expr {
+    <location:@L> <left:ConstantExpr> <op:AddOp> <right:ConstantAtom> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::BinOp {
+        end_location,
+        ast::ExprKind::BinOp {
             left: Box::new(left),
             op,
             right: Box::new(right),
         }
-    },
+    ),
 }
 
 LiteralPattern: ast::PatternKind = {
@@ -640,35 +605,32 @@ CapturePattern: ast::PatternKind = {
 }
 
 MatchName: ast::Expr = {
-    <location:@L> <name:Identifier> <end_location:@R> => ast::Expr {
+    <location:@L> <name:Identifier> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load },
-    },
+        end_location,
+        ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load },
+    ),
 }
 
 MatchNameOrAttr: ast::Expr = {
-    <location:@L> <name:MatchName> "." <attr:Identifier> <end_location:@R> => ast::Expr {
+    <location:@L> <name:MatchName> "." <attr:Identifier> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Attribute {
+        end_location,
+        ast::ExprKind::Attribute {
             value: Box::new(name),
             attr,
             ctx: ast::ExprContext::Load,
         },
-    },
-    <location:@L> <e:MatchNameOrAttr> "." <attr:Identifier> <end_location:@R> => ast::Expr {
+    ),
+    <location:@L> <e:MatchNameOrAttr> "." <attr:Identifier> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Attribute {
+        end_location,
+        ast::ExprKind::Attribute {
             value: Box::new(e),
             attr,
             ctx: ast::ExprContext::Load,
         }
-    }
+    )
 }
 
 ValuePattern: ast::PatternKind = {
@@ -681,33 +643,30 @@ MappingKey: ast::Expr = {
     ConstantExpr,
     AddOpExpr,
     MatchNameOrAttr,
-    <location:@L> "None" <end_location:@R> => ast::Expr {
+    <location:@L> "None" <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Constant {
+        end_location,
+        ast::ExprKind::Constant {
             value: ast::Constant::None,
             kind: None,
         },
-    },
-    <location:@L> "True" <end_location:@R> => ast::Expr {
+    ),
+    <location:@L> "True" <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Constant {
+        end_location,
+        ast::ExprKind::Constant {
             value: true.into(),
             kind: None,
         },
-    },
-    <location:@L> "False" <end_location:@R> => ast::Expr {
+    ),
+    <location:@L> "False" <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Constant {
+        end_location,
+        ast::ExprKind::Constant {
             value: false.into(),
             kind: None,
         },
-    },
+    ),
     <location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?),
 }
 
@@ -844,24 +803,23 @@ IfStatement: ast::Stmt = {
             .or_else(|| s2.last().and_then(|last| last.4.last()))
             .or_else(|| body.last())
             .unwrap()
-            .end_location;
+            .end_location
+            .unwrap();
         // handle elif:
         for i in s2.into_iter().rev() {
-            let x = ast::Stmt {
-                custom: (),
-                location: i.0,
+            let x = ast::Stmt::new(
+                i.0,
                 end_location,
-                node: ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last },
-            };
+                ast::StmtKind::If { test: Box::new(i.2), body: i.4, orelse: last },
+            );
             last = vec![x];
         }
 
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
             end_location,
-            node: ast::StmtKind::If { test: Box::new(test), body, orelse: last }
-        }
+            ast::StmtKind::If { test: Box::new(test), body, orelse: last }
+        )
     },
 };
 
@@ -872,17 +830,17 @@ WhileStatement: ast::Stmt = {
             .last()
             .or_else(|| body.last())
             .unwrap()
-            .end_location;
-        ast::Stmt {
-            custom: (),
+            .end_location
+            .unwrap();
+        ast::Stmt::new(
             location,
             end_location,
-            node: ast::StmtKind::While {
+            ast::StmtKind::While {
                 test: Box::new(test),
                 body,
                 orelse
             },
-        }
+        )
     },
 };
 
@@ -913,21 +871,20 @@ TryStatement: ast::Stmt = {
         let finalbody = finally.map(|s| s.2).unwrap_or_default();
         let end_location = finalbody
             .last()
-            .map(|last| last.end_location)
-            .or_else(|| orelse.last().map(|last| last.end_location))
-            .or_else(|| handlers.last().map(|last| last.end_location))
+            .and_then(|last| last.end_location)
+            .or_else(|| orelse.last().and_then(|last| last.end_location))
+            .or_else(|| handlers.last().and_then(|last| last.end_location))
             .unwrap();
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
             end_location,
-            node: ast::StmtKind::Try {
+            ast::StmtKind::Try {
                 body,
                 handlers,
                 orelse,
                 finalbody,
             },
-        }
+        )
     },
     <location:@L> "try" ":" <body:Suite> <handlers:ExceptStarClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> <end_location:@R> => {
         let orelse = else_suite.map(|s| s.2).unwrap_or_default();
@@ -935,37 +892,35 @@ TryStatement: ast::Stmt = {
         let end_location = finalbody
             .last()
             .or_else(|| orelse.last())
-            .map(|last| last.end_location)
-            .or_else(|| handlers.last().map(|last| last.end_location))
+            .and_then(|last| last.end_location)
+            .or_else(|| handlers.last().and_then(|last| last.end_location))
             .unwrap();
-        ast::Stmt {
-            custom: (),
+        ast::Stmt::new(
             location,
             end_location,
-            node: ast::StmtKind::TryStar {
+            ast::StmtKind::TryStar {
                 body,
                 handlers,
                 orelse,
                 finalbody,
             },
-        }
+        )
     },
     <location:@L> "try" ":" <body:Suite> <finally:("finally" ":" Suite)> => {
         let handlers = vec![];
         let orelse = vec![];
         let finalbody = finally.2;
-        let end_location = finalbody.last().unwrap().end_location;
-        ast::Stmt {
-            custom: (),
+        let end_location = finalbody.last().unwrap().end_location.unwrap();
+        ast::Stmt::new(
             location,
             end_location,
-            node: ast::StmtKind::Try {
+            ast::StmtKind::Try {
                 body,
                 handlers,
                 orelse,
                 finalbody,
             },
-        }
+        )
     },
 };
 
@@ -1242,19 +1197,18 @@ ClassDef: ast::Stmt = {
             Some((_, arg, _)) => (arg.args, arg.keywords),
             None => (vec![], vec![]),
         };
-        let end_location = body.last().unwrap().end_location;
-        ast::Stmt {
-            custom: (),
+        let end_location = body.last().unwrap().end_location.unwrap();
+        ast::Stmt::new(
             location,
             end_location,
-            node: ast::StmtKind::ClassDef {
+            ast::StmtKind::ClassDef {
                 name,
                 bases,
                 keywords,
                 body,
                 decorator_list,
             },
-        }
+        )
     },
 };
 
@@ -1266,31 +1220,28 @@ Decorator: ast::Expr = {
 };
 
 YieldExpr: ast::Expr = {
-    <location:@L> "yield" <value:TestList?> <end_location:@R> => ast::Expr {
+    <location:@L> "yield" <value:TestList?> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Yield { value: value.map(Box::new) }
-    },
-    <location:@L> "yield" "from" <e:Test<"all">> <end_location:@R> => ast::Expr {
+        end_location,
+        ast::ExprKind::Yield { value: value.map(Box::new) }
+    ),
+    <location:@L> "yield" "from" <e:Test<"all">> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::YieldFrom { value: Box::new(e) }
-    },
+        end_location,
+        ast::ExprKind::YieldFrom { value: Box::new(e) }
+    ),
 };
 
 Test<Goal>: ast::Expr = {
-    <location:@L> <body:OrTest<"all">> "if" <test:OrTest<"all">> "else" <orelse:Test<"all">> <end_location:@R> => ast::Expr {
+    <location:@L> <body:OrTest<"all">> "if" <test:OrTest<"all">> "else" <orelse:Test<"all">> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::IfExp {
+        end_location,
+        ast::ExprKind::IfExp {
             test: Box::new(test),
             body: Box::new(body),
             orelse: Box::new(orelse),
         }
-    },
+    ),
     OrTest<Goal>,
     LambdaDef,
 };
@@ -1334,15 +1285,14 @@ LambdaDef: ast::Expr = {
             }
         ))?;
 
-        Ok(ast::Expr {
+        Ok(ast::Expr::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::Lambda {
+            end_location,
+            ast::ExprKind::Lambda {
                 args: Box::new(p),
                 body: Box::new(body)
             }
-        })
+        ))
     }
 }
 
@@ -1350,12 +1300,11 @@ OrTest<Goal>: ast::Expr = {
     <location:@L> <e1:AndTest<"all">> <e2:("or" AndTest<"all">)+> <end_location:@R> => {
         let mut values = vec![e1];
         values.extend(e2.into_iter().map(|e| e.1));
-        ast::Expr {
+        ast::Expr::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values }
-        }
+            end_location,
+            ast::ExprKind::BoolOp { op: ast::Boolop::Or, values }
+        )
     },
     AndTest<Goal>,
 };
@@ -1364,35 +1313,32 @@ AndTest<Goal>: ast::Expr = {
     <location:@L> <e1:NotTest<"all">> <e2:("and" NotTest<"all">)+> <end_location:@R> => {
         let mut values = vec![e1];
         values.extend(e2.into_iter().map(|e| e.1));
-        ast::Expr {
+        ast::Expr::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values }
-        }
+            end_location,
+            ast::ExprKind::BoolOp { op: ast::Boolop::And, values }
+        )
     },
     NotTest<Goal>,
 };
 
 NotTest<Goal>: ast::Expr = {
-    <location:@L> "not" <e:NotTest<"all">> <end_location:@R> => ast::Expr {
+    <location:@L> "not" <e:NotTest<"all">> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::UnaryOp { operand: Box::new(e), op: ast::Unaryop::Not }
-    },
+        end_location,
+        ast::ExprKind::UnaryOp { operand: Box::new(e), op: ast::Unaryop::Not }
+    ),
     Comparison<Goal>,
 };
 
 Comparison<Goal>: ast::Expr = {
     <location:@L> <left:Expression<"all">> <comparisons:(CompOp Expression<"all">)+> <end_location:@R> => {
         let (ops, comparators) = comparisons.into_iter().unzip();
-        ast::Expr {
+        ast::Expr::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::Compare { left: Box::new(left), ops, comparators }
-        }
+            end_location,
+            ast::ExprKind::Compare { left: Box::new(left), ops, comparators }
+        )
     },
     Expression<Goal>,
 };
@@ -1411,42 +1357,38 @@ CompOp: ast::Cmpop = {
 };
 
 Expression<Goal>: ast::Expr = {
-    <location:@L> <e1:Expression<"all">> "|" <e2:XorExpression<"all">> <end_location:@R> => ast::Expr {
+    <location:@L> <e1:Expression<"all">> "|" <e2:XorExpression<"all">> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) }
-    },
+        end_location,
+        ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) }
+    ),
     XorExpression<Goal>,
 };
 
 XorExpression<Goal>: ast::Expr = {
-    <location:@L> <e1:XorExpression<"all">> "^" <e2:AndExpression<"all">> <end_location:@R> => ast::Expr {
+    <location:@L> <e1:XorExpression<"all">> "^" <e2:AndExpression<"all">> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) }
-    },
+        end_location,
+        ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) }
+    ),
     AndExpression<Goal>,
 };
 
 AndExpression<Goal>: ast::Expr = {
-    <location:@L> <e1:AndExpression<"all">> "&" <e2:ShiftExpression<"all">> <end_location:@R> => ast::Expr {
+    <location:@L> <e1:AndExpression<"all">> "&" <e2:ShiftExpression<"all">> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) }
-    },
+        end_location,
+        ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) }
+    ),
     ShiftExpression<Goal>,
 };
 
 ShiftExpression<Goal>: ast::Expr = {
-    <location:@L> <e1:ShiftExpression<"all">> <op:ShiftOp> <e2:ArithmeticExpression<"all">> <end_location:@R> => ast::Expr {
+    <location:@L> <e1:ShiftExpression<"all">> <op:ShiftOp> <e2:ArithmeticExpression<"all">> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) }
-    },
+        end_location,
+        ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) }
+    ),
     ArithmeticExpression<Goal>,
 };
 
@@ -1456,12 +1398,11 @@ ShiftOp: ast::Operator = {
 };
 
 ArithmeticExpression<Goal>: ast::Expr = {
-    <location:@L> <a:ArithmeticExpression<"all">> <op:AddOp> <b:Term<"all">> <end_location:@R> => ast::Expr {
+    <location:@L> <a:ArithmeticExpression<"all">> <op:AddOp> <b:Term<"all">> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
-    },
+        end_location,
+        ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
+    ),
     Term<Goal>,
 };
 
@@ -1471,12 +1412,11 @@ AddOp: ast::Operator = {
 };
 
 Term<Goal>: ast::Expr = {
-    <location:@L> <a:Term<"all">> <op:MulOp> <b:Factor<"all">> <end_location:@R> => ast::Expr {
+    <location:@L> <a:Term<"all">> <op:MulOp> <b:Factor<"all">> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
-    },
+        end_location,
+        ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) }
+    ),
     Factor<Goal>,
 };
 
@@ -1489,12 +1429,11 @@ MulOp: ast::Operator = {
 };
 
 Factor<Goal>: ast::Expr = {
-    <location:@L> <op:UnaryOp> <e:Factor<"all">> <end_location:@R>  => ast::Expr {
+    <location:@L> <op:UnaryOp> <e:Factor<"all">> <end_location:@R>  => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::UnaryOp { operand: Box::new(e), op }
-    },
+        end_location,
+        ast::ExprKind::UnaryOp { operand: Box::new(e), op }
+    ),
     Power<Goal>,
 };
 
@@ -1505,23 +1444,21 @@ UnaryOp: ast::Unaryop = {
 };
 
 Power<Goal>: ast::Expr = {
-    <location:@L> <e:AtomExpr<"all">> "**" <b:Factor<"all">> <end_location:@R> => ast::Expr {
+    <location:@L> <e:AtomExpr<"all">> "**" <b:Factor<"all">> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) }
-    },
+        end_location,
+        ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) }
+    ),
     AtomExpr<Goal>,
 };
 
 AtomExpr<Goal>: ast::Expr = {
     <location:@L> "await" <atom:AtomExpr2<"all">> <end_location:@R> => {
-        ast::Expr {
+        ast::Expr::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::Await { value: Box::new(atom) }
-        }
+            end_location,
+            ast::ExprKind::Await { value: Box::new(atom) }
+        )
     },
     AtomExpr2<Goal>,
 }
@@ -1529,25 +1466,22 @@ AtomExpr<Goal>: ast::Expr = {
 AtomExpr2<Goal>: ast::Expr = {
     Atom<Goal>,
     <location:@L> <f:AtomExpr2<"all">> "(" <a:ArgumentList> ")" <end_location:@R> => {
-        ast::Expr {
+        ast::Expr::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::Call { func: Box::new(f), args: a.args, keywords: a.keywords }
-        }
+            end_location,
+            ast::ExprKind::Call { func: Box::new(f), args: a.args, keywords: a.keywords }
+        )
     },
-    <location:@L> <e:AtomExpr2<"all">> "[" <s:SubscriptList> "]" <end_location:@R> => ast::Expr {
+    <location:@L> <e:AtomExpr2<"all">> "[" <s:SubscriptList> "]" <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Subscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load }
-    },
-    <location:@L> <e:AtomExpr2<"all">> "." <attr:Identifier> <end_location:@R> => ast::Expr {
+        end_location,
+        ast::ExprKind::Subscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load }
+    ),
+    <location:@L> <e:AtomExpr2<"all">> "." <attr:Identifier> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Attribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load }
-    },
+        end_location,
+        ast::ExprKind::Attribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load }
+    ),
 };
 
 SubscriptList: ast::Expr = {
@@ -1560,12 +1494,11 @@ SubscriptList: ast::Expr = {
                 dims.push(x.1)
             }
 
-            ast::Expr {
+            ast::Expr::new(
                 location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::ExprKind::Tuple { elts: dims, ctx: ast::ExprContext::Load },
-            }
+                end_location,
+                ast::ExprKind::Tuple { elts: dims, ctx: ast::ExprContext::Load },
+            )
         }
     }
 };
@@ -1576,12 +1509,11 @@ Subscript: ast::Expr = {
         let lower = e1.map(Box::new);
         let upper = e2.map(Box::new);
         let step = e3.flatten().map(Box::new);
-        ast::Expr {
+        ast::Expr::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::Slice { lower, upper, step }
-        }
+            end_location,
+            ast::ExprKind::Slice { lower, upper, step }
+        )
     }
 };
 
@@ -1591,34 +1523,30 @@ SliceOp: Option<ast::Expr> = {
 
 Atom<Goal>: ast::Expr = {
     <location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?),
-    <location:@L> <value:Constant> <end_location:@R> => ast::Expr {
+    <location:@L> <value:Constant> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Constant { value, kind: None }
-    },
-    <location:@L> <name:Identifier> <end_location:@R> => ast::Expr {
+        end_location,
+        ast::ExprKind::Constant { value, kind: None }
+    ),
+    <location:@L> <name:Identifier> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load }
-    },
+        end_location,
+        ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load }
+    ),
     <location:@L> "[" <e:ListLiteralValues?> "]"<end_location:@R>  => {
         let elts = e.unwrap_or_default();
-        ast::Expr {
+        ast::Expr::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::List { elts, ctx: ast::ExprContext::Load }
-        }
+            end_location,
+            ast::ExprKind::List { elts, ctx: ast::ExprContext::Load }
+        )
     },
     <location:@L> "[" <elt:TestOrStarNamedExpr> <generators:CompFor> "]" <end_location:@R> => {
-        ast::Expr {
+        ast::Expr::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::ListComp { elt: Box::new(elt), generators }
-        }
+            end_location,
+            ast::ExprKind::ListComp { elt: Box::new(elt), generators }
+        )
     },
     <location:@L> "(" <elts:OneOrMore<Test<"all">>> <trailing_comma:","?> ")" <end_location:@R> if Goal != "no-withitems" => {
         if elts.len() == 1 && trailing_comma.is_none() {
@@ -1656,12 +1584,11 @@ Atom<Goal>: ast::Expr = {
     ),
     "(" <e:YieldExpr> ")" => e,
     <location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" <end_location:@R> => {
-        ast::Expr {
+        ast::Expr::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::GeneratorExp { elt: Box::new(elt), generators }
-        }
+            end_location,
+            ast::ExprKind::GeneratorExp { elt: Box::new(elt), generators }
+        )
     },
     "(" <location:@L> "**" <e:Expression<"all">> ")" <end_location:@R> =>? {
         Err(LexicalError{
@@ -1675,38 +1602,34 @@ Atom<Goal>: ast::Expr = {
             .into_iter()
             .map(|(k, v)| (k.map(|x| *x), v))
             .unzip();
-        ast::Expr {
+        ast::Expr::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::Dict { keys, values }
-        }
+            end_location,
+            ast::ExprKind::Dict { keys, values }
+        )
     },
     <location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" <end_location:@R> => {
-        ast::Expr {
+        ast::Expr::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::DictComp {
+            end_location,
+            ast::ExprKind::DictComp {
                 key: Box::new(e1.0),
                 value: Box::new(e1.1),
                 generators,
             }
-        }
+        )
     },
-    <location:@L> "{" <elts:SetLiteralValues> "}" <end_location:@R> => ast::Expr {
+    <location:@L> "{" <elts:SetLiteralValues> "}" <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Set { elts }
-    },
+        end_location,
+        ast::ExprKind::Set { elts }
+    ),
     <location:@L> "{" <elt:NamedExpressionTest> <generators:CompFor> "}" <end_location:@R> => {
-        ast::Expr {
+        ast::Expr::new(
             location,
-            end_location: Some(end_location),
-            custom: (),
-            node: ast::ExprKind::SetComp { elt: Box::new(elt), generators }
-        }
+            end_location,
+            ast::ExprKind::SetComp { elt: Box::new(elt), generators }
+        )
     },
     <location:@L> "True" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: true.into(), kind: None }),
     <location:@L> "False" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: false.into(), kind: None }),
@@ -1762,24 +1685,22 @@ GenericList<Element>: ast::Expr = {
         if elts.len() == 1 && trailing_comma.is_none() {
             elts.into_iter().next().unwrap()
         } else {
-            ast::Expr {
+            ast::Expr::new(
                 location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load }
-            }
+                end_location,
+                ast::ExprKind::Tuple { elts, ctx: ast::ExprContext::Load }
+            )
         }
     }
 }
 
 // Test
 StarExpr: ast::Expr = {
-    <location:@L> "*" <e:Expression<"all">> <end_location:@R> => ast::Expr {
+    <location:@L> "*" <e:Expression<"all">> <end_location:@R> => ast::Expr::new(
         location,
-        end_location: Some(end_location),
-        custom: (),
-        node: ast::ExprKind::Starred { value: Box::new(e), ctx: ast::ExprContext::Load },
-    }
+        end_location,
+        ast::ExprKind::Starred { value: Box::new(e), ctx: ast::ExprContext::Load },
+    )
 };
 
 // Comprehensions:
@@ -1810,15 +1731,14 @@ ArgumentList: ArgumentList = {
 FunctionArgument: (Option<(ast::Location, ast::Location, Option<String>)>, ast::Expr) = {
     <location:@L> <e:NamedExpressionTest> <c:CompFor?> <end_location:@R> => {
         let expr = match c {
-            Some(c) => ast::Expr {
+            Some(c) => ast::Expr::new(
                 location,
-                end_location: Some(end_location),
-                custom: (),
-                node: ast::ExprKind::GeneratorExp {
+                end_location,
+                ast::ExprKind::GeneratorExp {
                     elt: Box::new(e),
                     generators: c,
                 }
-            },
+            ),
             None => e,
         };
         (None, expr)

From 31fd625321493473551b72b19c68aa70f6798bf1 Mon Sep 17 00:00:00 2001
From: Micha Reiser <micha@reiser.io>
Date: Wed, 26 Apr 2023 10:24:34 -0600
Subject: [PATCH 455/459] Add `Located::start`, `Located::end` and impl `Deref`

---
 ast/asdl_rs.py         | 18 ++++++++++++++
 ast/src/ast_gen.rs     | 18 ++++++++++++++
 parser/python.lalrpop  | 55 ++++++++++++++++++------------------------
 parser/src/function.rs | 10 ++++----
 4 files changed, 65 insertions(+), 36 deletions(-)

diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py
index 8caeca90..c9b819da 100755
--- a/ast/asdl_rs.py
+++ b/ast/asdl_rs.py
@@ -671,6 +671,24 @@ def write_ast_def(mod, typeinfo, f):
             pub fn new(location: Location, end_location: Location, node: T) -> Self {
                 Self { location, end_location: Some(end_location), custom: (), node }
             }
+            
+            pub const fn start(&self) -> Location {
+                self.location
+            }
+            
+            /// Returns the node's [`end_location`](Located::end_location) or [`location`](Located::start) if 
+            /// [`end_location`](Located::end_location) is `None`.
+            pub fn end(&self) -> Location {
+                self.end_location.unwrap_or(self.location)
+            }      
+        }
+        
+        impl<T, U> std::ops::Deref for Located<T, U> {
+            type Target = T;
+            
+            fn deref(&self) -> &Self::Target {
+                &self.node
+            }
         }
         \n
     """.lstrip()
diff --git a/ast/src/ast_gen.rs b/ast/src/ast_gen.rs
index c11fd87d..6771dd0e 100644
--- a/ast/src/ast_gen.rs
+++ b/ast/src/ast_gen.rs
@@ -24,6 +24,24 @@ impl<T> Located<T> {
             node,
         }
     }
+
+    pub const fn start(&self) -> Location {
+        self.location
+    }
+
+    /// Returns the node's [`end_location`](Located::end_location) or [`location`](Located::start) if
+    /// [`end_location`](Located::end_location) is `None`.
+    pub fn end(&self) -> Location {
+        self.end_location.unwrap_or(self.location)
+    }
+}
+
+impl<T, U> std::ops::Deref for Located<T, U> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        &self.node
+    }
 }
 
 #[derive(Clone, Debug, PartialEq)]
diff --git a/parser/python.lalrpop b/parser/python.lalrpop
index 32c08809..45113a3c 100644
--- a/parser/python.lalrpop
+++ b/parser/python.lalrpop
@@ -348,8 +348,7 @@ MatchStatement: ast::Stmt = {
             .body
             .last()
             .unwrap()
-            .end_location
-            .unwrap();
+            .end();
         ast::Stmt::new(
             location,
             end_location,
@@ -366,8 +365,7 @@ MatchStatement: ast::Stmt = {
             .body
             .last()
             .unwrap()
-            .end_location
-            .unwrap();
+            .end();
         ast::Stmt::new(
             location,
             end_location,
@@ -384,8 +382,7 @@ MatchStatement: ast::Stmt = {
             .body
             .last()
             .unwrap()
-            .end_location
-            .unwrap();
+            .end();
         let mut subjects = subjects;
         subjects.insert(0, subject);
         ast::Stmt::new(
@@ -803,8 +800,7 @@ IfStatement: ast::Stmt = {
             .or_else(|| s2.last().and_then(|last| last.4.last()))
             .or_else(|| body.last())
             .unwrap()
-            .end_location
-            .unwrap();
+            .end();
         // handle elif:
         for i in s2.into_iter().rev() {
             let x = ast::Stmt::new(
@@ -830,8 +826,7 @@ WhileStatement: ast::Stmt = {
             .last()
             .or_else(|| body.last())
             .unwrap()
-            .end_location
-            .unwrap();
+            .end();
         ast::Stmt::new(
             location,
             end_location,
@@ -851,8 +846,7 @@ ForStatement: ast::Stmt = {
             .last()
             .or_else(|| body.last())
             .unwrap()
-            .end_location
-            .unwrap();
+            .end();
         let target = Box::new(set_context(target, ast::ExprContext::Store));
         let iter = Box::new(iter);
         let type_comment = None;
@@ -871,9 +865,9 @@ TryStatement: ast::Stmt = {
         let finalbody = finally.map(|s| s.2).unwrap_or_default();
         let end_location = finalbody
             .last()
-            .and_then(|last| last.end_location)
-            .or_else(|| orelse.last().and_then(|last| last.end_location))
-            .or_else(|| handlers.last().and_then(|last| last.end_location))
+            .map(|last| last.end())
+            .or_else(|| orelse.last().map(|last| last.end()))
+            .or_else(|| handlers.last().map(|last| last.end()))
             .unwrap();
         ast::Stmt::new(
             location,
@@ -892,8 +886,8 @@ TryStatement: ast::Stmt = {
         let end_location = finalbody
             .last()
             .or_else(|| orelse.last())
-            .and_then(|last| last.end_location)
-            .or_else(|| handlers.last().and_then(|last| last.end_location))
+            .map(|last| last.end())
+            .or_else(|| handlers.last().map(|last| last.end()))
             .unwrap();
         ast::Stmt::new(
             location,
@@ -910,7 +904,7 @@ TryStatement: ast::Stmt = {
         let handlers = vec![];
         let orelse = vec![];
         let finalbody = finally.2;
-        let end_location = finalbody.last().unwrap().end_location.unwrap();
+        let end_location = finalbody.last().unwrap().end();
         ast::Stmt::new(
             location,
             end_location,
@@ -926,7 +920,7 @@ TryStatement: ast::Stmt = {
 
 ExceptStarClause: ast::Excepthandler = {
     <location:@L> "except" "*" <typ:Test<"all">> ":" <body:Suite> => {
-        let end_location = body.last().unwrap().end_location.unwrap();
+        let end_location = body.last().unwrap().end();
         ast::Excepthandler::new(
             location,
             end_location,
@@ -938,7 +932,7 @@ ExceptStarClause: ast::Excepthandler = {
         )
     },
     <location:@L> "except" "*" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
-        let end_location = body.last().unwrap().end_location.unwrap();
+        let end_location = body.last().unwrap().end();
         ast::Excepthandler::new(
             location,
             end_location,
@@ -954,7 +948,7 @@ ExceptStarClause: ast::Excepthandler = {
 
 ExceptClause: ast::Excepthandler = {
     <location:@L> "except" <typ:Test<"all">?> ":" <body:Suite> => {
-        let end_location = body.last().unwrap().end_location.unwrap();
+        let end_location = body.last().unwrap().end();
         ast::Excepthandler::new(
             location,
             end_location,
@@ -966,7 +960,7 @@ ExceptClause: ast::Excepthandler = {
         )
     },
     <location:@L> "except" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
-        let end_location = body.last().unwrap().end_location.unwrap();
+        let end_location = body.last().unwrap().end();
         ast::Excepthandler::new(
             location,
             end_location,
@@ -981,7 +975,7 @@ ExceptClause: ast::Excepthandler = {
 
 WithStatement: ast::Stmt = {
     <location:@L> <is_async:"async"?> "with" <items:WithItems> ":" <body:Suite> => {
-        let end_location = body.last().unwrap().end_location.unwrap();
+        let end_location = body.last().unwrap().end();
         let type_comment = None;
         let node = if is_async.is_some() {
             ast::StmtKind::AsyncWith { items, body, type_comment }
@@ -1022,7 +1016,7 @@ FuncDef: ast::Stmt = {
     <decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test<"all">)?> ":" <body:Suite>  => {
         let args = Box::new(args);
         let returns = r.map(|x| Box::new(x.1));
-        let end_location = body.last().unwrap().end_location.unwrap();
+        let end_location = body.last().unwrap().end();
         let type_comment = None;
         let node = if is_async.is_some() {
             ast::StmtKind::AsyncFunctionDef { name, args, body, decorator_list, returns, type_comment }
@@ -1197,7 +1191,7 @@ ClassDef: ast::Stmt = {
             Some((_, arg, _)) => (arg.args, arg.keywords),
             None => (vec![], vec![]),
         };
-        let end_location = body.last().unwrap().end_location.unwrap();
+        let end_location = body.last().unwrap().end();
         ast::Stmt::new(
             location,
             end_location,
@@ -1253,11 +1247,10 @@ NamedExpressionTest: ast::Expr = {
 
 NamedExpression: ast::Expr = {
     <location:@L> <id:Identifier> <end_location:@R> ":=" <value:Test<"all">> => {
-        ast::Expr {
+        ast::Expr::new(
             location,
-            end_location: value.end_location,
-            custom: (),
-            node: ast::ExprKind::NamedExpr {
+            value.end(),
+            ast::ExprKind::NamedExpr {
                 target: Box::new(ast::Expr::new(
                     location,
                     end_location,
@@ -1265,7 +1258,7 @@ NamedExpression: ast::Expr = {
                 )),
                 value: Box::new(value),
             }
-        }
+        )
     },
 };
 
@@ -1564,7 +1557,7 @@ Atom<Goal>: ast::Expr = {
             if matches!(mid.node, ast::ExprKind::Starred { .. }) {
                 Err(LexicalError{
                     error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
-                    location: mid.location,
+                    location: mid.start(),
                 })?
             }
             Ok(mid)
diff --git a/parser/src/function.rs b/parser/src/function.rs
index 0f580e77..17b9882b 100644
--- a/parser/src/function.rs
+++ b/parser/src/function.rs
@@ -35,12 +35,12 @@ pub(crate) fn validate_arguments(
 
     let mut all_arg_names = FxHashSet::with_hasher(Default::default());
     for arg in all_args {
-        let arg_name = &arg.node.arg;
+        let arg_name = &arg.arg;
         // Check for duplicate arguments in the function definition.
         if !all_arg_names.insert(arg_name) {
             return Err(LexicalError {
                 error: LexicalErrorType::DuplicateArgumentError(arg_name.to_string()),
-                location: arg.location,
+                location: arg.start(),
             });
         }
     }
@@ -64,7 +64,7 @@ pub(crate) fn parse_params(
             // have defaults.
             return Err(LexicalError {
                 error: LexicalErrorType::DefaultArgumentError,
-                location: name.location,
+                location: name.start(),
             });
         }
         Ok(())
@@ -126,14 +126,14 @@ pub(crate) fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentLis
                 if !keywords.is_empty() && !is_starred(&value) {
                     return Err(LexicalError {
                         error: LexicalErrorType::PositionalArgumentError,
-                        location: value.location,
+                        location: value.start(),
                     });
                 // Allow starred arguments after keyword arguments but
                 // not after double-starred arguments.
                 } else if double_starred {
                     return Err(LexicalError {
                         error: LexicalErrorType::UnpackedArgumentError,
-                        location: value.location,
+                        location: value.start(),
                     });
                 }
 

From bbbe25336a117e0b036e8a5678519d93e79feedb Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 3 May 2023 03:25:40 +0900
Subject: [PATCH 456/459] Refactor common::str::repr using common::escape

---
 ast/src/constant.rs | 5 ++++-
 ast/src/unparse.rs  | 3 ++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/ast/src/constant.rs b/ast/src/constant.rs
index 6099272e..7b40fb1f 100644
--- a/ast/src/constant.rs
+++ b/ast/src/constant.rs
@@ -41,7 +41,10 @@ impl std::fmt::Display for Constant {
         match self {
             Constant::None => f.pad("None"),
             Constant::Bool(b) => f.pad(if *b { "True" } else { "False" }),
-            Constant::Str(s) => rustpython_common::str::repr(s).fmt(f),
+            Constant::Str(s) => {
+                use rustpython_common::escape::Escape;
+                rustpython_common::escape::UnicodeEscape::new_repr(s.as_str()).write_quoted(f)
+            }
             Constant::Bytes(b) => {
                 f.pad(&rustpython_common::bytes::repr(b).map_err(|_err| std::fmt::Error)?)
             }
diff --git a/ast/src/unparse.rs b/ast/src/unparse.rs
index 081c2a92..1af7bf2e 100644
--- a/ast/src/unparse.rs
+++ b/ast/src/unparse.rs
@@ -509,9 +509,10 @@ impl<'a> Unparser<'a> {
         if is_spec {
             self.unparse_fstring_body(values, is_spec)
         } else {
+            use rustpython_common::escape::Escape;
             self.p("f")?;
             let body = to_string_fmt(|f| Unparser::new(f).unparse_fstring_body(values, is_spec));
-            fmt::Display::fmt(&rustpython_common::str::repr(&body), &mut self.f)
+            rustpython_common::escape::UnicodeEscape::new_repr(&body).write_quoted(&mut self.f)
         }
     }
 }

From e09b23d3fb66c0cccbeabe61fa86eccb4f0489b5 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Wed, 3 May 2023 15:01:52 +0900
Subject: [PATCH 457/459] Refactor common::bytes::repr using common::escape

---
 ast/src/constant.rs | 11 ++++++-----
 ast/src/unparse.rs  |  5 +++--
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/ast/src/constant.rs b/ast/src/constant.rs
index 7b40fb1f..d81d7fbe 100644
--- a/ast/src/constant.rs
+++ b/ast/src/constant.rs
@@ -41,12 +41,13 @@ impl std::fmt::Display for Constant {
         match self {
             Constant::None => f.pad("None"),
             Constant::Bool(b) => f.pad(if *b { "True" } else { "False" }),
-            Constant::Str(s) => {
-                use rustpython_common::escape::Escape;
-                rustpython_common::escape::UnicodeEscape::new_repr(s.as_str()).write_quoted(f)
-            }
+            Constant::Str(s) => rustpython_common::escape::UnicodeEscape::new_repr(s.as_str())
+                .str_repr()
+                .write(f),
             Constant::Bytes(b) => {
-                f.pad(&rustpython_common::bytes::repr(b).map_err(|_err| std::fmt::Error)?)
+                let escape = rustpython_common::escape::AsciiEscape::new_repr(b);
+                let repr = escape.bytes_repr().to_string().unwrap();
+                f.pad(&repr)
             }
             Constant::Int(i) => i.fmt(f),
             Constant::Tuple(tup) => {
diff --git a/ast/src/unparse.rs b/ast/src/unparse.rs
index 1af7bf2e..e7e1bced 100644
--- a/ast/src/unparse.rs
+++ b/ast/src/unparse.rs
@@ -509,10 +509,11 @@ impl<'a> Unparser<'a> {
         if is_spec {
             self.unparse_fstring_body(values, is_spec)
         } else {
-            use rustpython_common::escape::Escape;
             self.p("f")?;
             let body = to_string_fmt(|f| Unparser::new(f).unparse_fstring_body(values, is_spec));
-            rustpython_common::escape::UnicodeEscape::new_repr(&body).write_quoted(&mut self.f)
+            rustpython_common::escape::UnicodeEscape::new_repr(&body)
+                .str_repr()
+                .write(&mut self.f)
         }
     }
 }

From a0a2f65bf006fa0b5bcd06fefc1d9bf6731044c5 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Fri, 5 May 2023 20:41:18 +0900
Subject: [PATCH 458/459] rustpython-literal

---
 ast/Cargo.toml        |   4 +-
 ast/src/constant.rs   |   8 +-
 ast/src/unparse.rs    |   2 +-
 literal/Cargo.toml    |  18 ++
 literal/src/char.rs   |  15 ++
 literal/src/escape.rs | 414 ++++++++++++++++++++++++++++++++++++++++++
 literal/src/float.rs  | 310 +++++++++++++++++++++++++++++++
 literal/src/format.rs |   5 +
 literal/src/lib.rs    |   4 +
 9 files changed, 773 insertions(+), 7 deletions(-)
 create mode 100644 literal/Cargo.toml
 create mode 100644 literal/src/char.rs
 create mode 100644 literal/src/escape.rs
 create mode 100644 literal/src/float.rs
 create mode 100644 literal/src/format.rs
 create mode 100644 literal/src/lib.rs

diff --git a/ast/Cargo.toml b/ast/Cargo.toml
index bb280f48..5de211dd 100644
--- a/ast/Cargo.toml
+++ b/ast/Cargo.toml
@@ -11,10 +11,10 @@ license = "MIT"
 default = ["constant-optimization", "fold"]
 constant-optimization = ["fold"]
 fold = []
-unparse = ["rustpython-common"]
+unparse = ["rustpython-literal"]
 
 [dependencies]
 rustpython-compiler-core = { path = "../core", version = "0.2.0" }
-rustpython-common = { path = "../../common", version = "0.2.0", optional = true }
+rustpython-literal = { path = "../literal", version = "0.2.0", optional = true }
 
 num-bigint = { workspace = true }
diff --git a/ast/src/constant.rs b/ast/src/constant.rs
index d81d7fbe..3514e521 100644
--- a/ast/src/constant.rs
+++ b/ast/src/constant.rs
@@ -35,17 +35,17 @@ impl From<BigInt> for Constant {
     }
 }
 
-#[cfg(feature = "rustpython-common")]
+#[cfg(feature = "rustpython-literal")]
 impl std::fmt::Display for Constant {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Constant::None => f.pad("None"),
             Constant::Bool(b) => f.pad(if *b { "True" } else { "False" }),
-            Constant::Str(s) => rustpython_common::escape::UnicodeEscape::new_repr(s.as_str())
+            Constant::Str(s) => rustpython_literal::escape::UnicodeEscape::new_repr(s.as_str())
                 .str_repr()
                 .write(f),
             Constant::Bytes(b) => {
-                let escape = rustpython_common::escape::AsciiEscape::new_repr(b);
+                let escape = rustpython_literal::escape::AsciiEscape::new_repr(b);
                 let repr = escape.bytes_repr().to_string().unwrap();
                 f.pad(&repr)
             }
@@ -64,7 +64,7 @@ impl std::fmt::Display for Constant {
                     f.write_str(")")
                 }
             }
-            Constant::Float(fp) => f.pad(&rustpython_common::float_ops::to_string(*fp)),
+            Constant::Float(fp) => f.pad(&rustpython_literal::float::to_string(*fp)),
             Constant::Complex { real, imag } => {
                 if *real == 0.0 {
                     write!(f, "{imag}j")
diff --git a/ast/src/unparse.rs b/ast/src/unparse.rs
index e7e1bced..40570fc7 100644
--- a/ast/src/unparse.rs
+++ b/ast/src/unparse.rs
@@ -511,7 +511,7 @@ impl<'a> Unparser<'a> {
         } else {
             self.p("f")?;
             let body = to_string_fmt(|f| Unparser::new(f).unparse_fstring_body(values, is_spec));
-            rustpython_common::escape::UnicodeEscape::new_repr(&body)
+            rustpython_literal::escape::UnicodeEscape::new_repr(&body)
                 .str_repr()
                 .write(&mut self.f)
         }
diff --git a/literal/Cargo.toml b/literal/Cargo.toml
new file mode 100644
index 00000000..7afa9db3
--- /dev/null
+++ b/literal/Cargo.toml
@@ -0,0 +1,18 @@
+[package]
+name = "rustpython-literal"
+version = "0.2.0"
+description = "Common literal handling utilities mostly useful for unparse and repr."
+authors = ["RustPython Team"]
+edition = "2021"
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
+
+[dependencies]
+num-traits = { workspace = true }
+
+hexf-parse = "0.2.1"
+lexical-parse-float = { version = "0.8.0", features = ["format"] }
+unic-ucd-category = "0.9"
+
+[dev-dependencies]
+rand = { workspace = true }
diff --git a/literal/src/char.rs b/literal/src/char.rs
new file mode 100644
index 00000000..cd64f6df
--- /dev/null
+++ b/literal/src/char.rs
@@ -0,0 +1,15 @@
+use unic_ucd_category::GeneralCategory;
+
+/// According to python following categories aren't printable:
+/// * Cc (Other, Control)
+/// * Cf (Other, Format)
+/// * Cs (Other, Surrogate)
+/// * Co (Other, Private Use)
+/// * Cn (Other, Not Assigned)
+/// * Zl Separator, Line ('\u2028', LINE SEPARATOR)
+/// * Zp Separator, Paragraph ('\u2029', PARAGRAPH SEPARATOR)
+/// * Zs (Separator, Space) other than ASCII space('\x20').
+pub fn is_printable(c: char) -> bool {
+    let cat = GeneralCategory::of(c);
+    !(cat.is_other() || cat.is_separator())
+}
diff --git a/literal/src/escape.rs b/literal/src/escape.rs
new file mode 100644
index 00000000..de7a5143
--- /dev/null
+++ b/literal/src/escape.rs
@@ -0,0 +1,414 @@
+#[derive(Debug, Clone, Copy)]
+pub enum Quote {
+    Single,
+    Double,
+}
+
+impl Quote {
+    #[inline]
+    pub const fn swap(self) -> Quote {
+        match self {
+            Quote::Single => Quote::Double,
+            Quote::Double => Quote::Single,
+        }
+    }
+
+    #[inline]
+    pub const fn to_byte(&self) -> u8 {
+        match self {
+            Quote::Single => b'\'',
+            Quote::Double => b'"',
+        }
+    }
+
+    #[inline]
+    pub const fn to_char(&self) -> char {
+        match self {
+            Quote::Single => '\'',
+            Quote::Double => '"',
+        }
+    }
+}
+
+pub struct EscapeLayout {
+    pub quote: Quote,
+    pub len: Option<usize>,
+}
+
+pub trait Escape {
+    fn source_len(&self) -> usize;
+    fn layout(&self) -> &EscapeLayout;
+    fn changed(&self) -> bool {
+        self.layout().len != Some(self.source_len())
+    }
+
+    fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result;
+    fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result;
+    fn write_body(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        if self.changed() {
+            self.write_body_slow(formatter)
+        } else {
+            self.write_source(formatter)
+        }
+    }
+}
+
+/// Returns the outer quotes to use and the number of quotes that need to be
+/// escaped.
+pub(crate) const fn choose_quote(
+    single_count: usize,
+    double_count: usize,
+    preferred_quote: Quote,
+) -> (Quote, usize) {
+    let (primary_count, secondary_count) = match preferred_quote {
+        Quote::Single => (single_count, double_count),
+        Quote::Double => (double_count, single_count),
+    };
+
+    // always use primary unless we have primary but no seconday
+    let use_secondary = primary_count > 0 && secondary_count == 0;
+    if use_secondary {
+        (preferred_quote.swap(), secondary_count)
+    } else {
+        (preferred_quote, primary_count)
+    }
+}
+
+pub struct UnicodeEscape<'a> {
+    source: &'a str,
+    layout: EscapeLayout,
+}
+
+impl<'a> UnicodeEscape<'a> {
+    pub fn with_forced_quote(source: &'a str, quote: Quote) -> Self {
+        let layout = EscapeLayout { quote, len: None };
+        Self { source, layout }
+    }
+    pub fn new_repr(source: &'a str) -> Self {
+        let layout = Self::repr_layout(source, Quote::Single);
+        Self { source, layout }
+    }
+
+    pub fn str_repr<'r>(&'a self) -> StrRepr<'r, 'a> {
+        StrRepr(self)
+    }
+}
+
+pub struct StrRepr<'r, 'a>(&'r UnicodeEscape<'a>);
+
+impl StrRepr<'_, '_> {
+    pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        let quote = self.0.layout().quote.to_char();
+        formatter.write_char(quote)?;
+        self.0.write_body(formatter)?;
+        formatter.write_char(quote)
+    }
+
+    pub fn to_string(&self) -> Option<String> {
+        let mut s = String::with_capacity(self.0.layout().len?);
+        self.write(&mut s).unwrap();
+        Some(s)
+    }
+}
+
+impl std::fmt::Display for StrRepr<'_, '_> {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.write(formatter)
+    }
+}
+
+impl UnicodeEscape<'_> {
+    const REPR_RESERVED_LEN: usize = 2; // for quotes
+
+    pub fn repr_layout(source: &str, preferred_quote: Quote) -> EscapeLayout {
+        Self::output_layout_with_checker(source, preferred_quote, |a, b| {
+            Some((a as isize).checked_add(b as isize)? as usize)
+        })
+    }
+
+    fn output_layout_with_checker(
+        source: &str,
+        preferred_quote: Quote,
+        length_add: impl Fn(usize, usize) -> Option<usize>,
+    ) -> EscapeLayout {
+        let mut out_len = Self::REPR_RESERVED_LEN;
+        let mut single_count = 0;
+        let mut double_count = 0;
+
+        for ch in source.chars() {
+            let incr = match ch {
+                '\'' => {
+                    single_count += 1;
+                    1
+                }
+                '"' => {
+                    double_count += 1;
+                    1
+                }
+                c => Self::escaped_char_len(c),
+            };
+            let Some(new_len) = length_add(out_len, incr) else {
+                #[cold]
+                fn stop(single_count: usize, double_count: usize, preferred_quote: Quote) -> EscapeLayout {
+                    EscapeLayout { quote: choose_quote(single_count, double_count, preferred_quote).0, len: None }
+                }
+                return stop(single_count, double_count, preferred_quote);
+            };
+            out_len = new_len;
+        }
+
+        let (quote, num_escaped_quotes) = choose_quote(single_count, double_count, preferred_quote);
+        // we'll be adding backslashes in front of the existing inner quotes
+        let Some(out_len) = length_add(out_len, num_escaped_quotes) else {
+            return EscapeLayout { quote, len: None };
+        };
+
+        EscapeLayout {
+            quote,
+            len: Some(out_len - Self::REPR_RESERVED_LEN),
+        }
+    }
+
+    fn escaped_char_len(ch: char) -> usize {
+        match ch {
+            '\\' | '\t' | '\r' | '\n' => 2,
+            ch if ch < ' ' || ch as u32 == 0x7f => 4, // \xHH
+            ch if ch.is_ascii() => 1,
+            ch if crate::char::is_printable(ch) => {
+                // max = std::cmp::max(ch, max);
+                ch.len_utf8()
+            }
+            ch if (ch as u32) < 0x100 => 4,   // \xHH
+            ch if (ch as u32) < 0x10000 => 6, // \uHHHH
+            _ => 10,                          // \uHHHHHHHH
+        }
+    }
+
+    fn write_char(
+        ch: char,
+        quote: Quote,
+        formatter: &mut impl std::fmt::Write,
+    ) -> std::fmt::Result {
+        match ch {
+            '\n' => formatter.write_str("\\n"),
+            '\t' => formatter.write_str("\\t"),
+            '\r' => formatter.write_str("\\r"),
+            // these 2 branches *would* be handled below, but we shouldn't have to do a
+            // unicodedata lookup just for ascii characters
+            '\x20'..='\x7e' => {
+                // printable ascii range
+                if ch == quote.to_char() || ch == '\\' {
+                    formatter.write_char('\\')?;
+                }
+                formatter.write_char(ch)
+            }
+            ch if ch.is_ascii() => {
+                write!(formatter, "\\x{:02x}", ch as u8)
+            }
+            ch if crate::char::is_printable(ch) => formatter.write_char(ch),
+            '\0'..='\u{ff}' => {
+                write!(formatter, "\\x{:02x}", ch as u32)
+            }
+            '\0'..='\u{ffff}' => {
+                write!(formatter, "\\u{:04x}", ch as u32)
+            }
+            _ => {
+                write!(formatter, "\\U{:08x}", ch as u32)
+            }
+        }
+    }
+}
+
+impl<'a> Escape for UnicodeEscape<'a> {
+    fn source_len(&self) -> usize {
+        self.source.len()
+    }
+
+    fn layout(&self) -> &EscapeLayout {
+        &self.layout
+    }
+
+    fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        formatter.write_str(self.source)
+    }
+
+    #[cold]
+    fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        for ch in self.source.chars() {
+            Self::write_char(ch, self.layout().quote, formatter)?;
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod unicode_escapse_tests {
+    use super::*;
+
+    #[test]
+    fn changed() {
+        fn test(s: &str) -> bool {
+            UnicodeEscape::new_repr(s).changed()
+        }
+        assert!(!test("hello"));
+        assert!(!test("'hello'"));
+        assert!(!test("\"hello\""));
+
+        assert!(test("'\"hello"));
+        assert!(test("hello\n"));
+    }
+}
+
+pub struct AsciiEscape<'a> {
+    source: &'a [u8],
+    layout: EscapeLayout,
+}
+
+impl<'a> AsciiEscape<'a> {
+    pub fn new(source: &'a [u8], layout: EscapeLayout) -> Self {
+        Self { source, layout }
+    }
+    pub fn with_forced_quote(source: &'a [u8], quote: Quote) -> Self {
+        let layout = EscapeLayout { quote, len: None };
+        Self { source, layout }
+    }
+    pub fn new_repr(source: &'a [u8]) -> Self {
+        let layout = Self::repr_layout(source, Quote::Single);
+        Self { source, layout }
+    }
+
+    pub fn bytes_repr<'r>(&'a self) -> BytesRepr<'r, 'a> {
+        BytesRepr(self)
+    }
+}
+
+impl AsciiEscape<'_> {
+    pub fn repr_layout(source: &[u8], preferred_quote: Quote) -> EscapeLayout {
+        Self::output_layout_with_checker(source, preferred_quote, 3, |a, b| {
+            Some((a as isize).checked_add(b as isize)? as usize)
+        })
+    }
+
+    pub fn named_repr_layout(source: &[u8], name: &str) -> EscapeLayout {
+        Self::output_layout_with_checker(source, Quote::Single, name.len() + 2 + 3, |a, b| {
+            Some((a as isize).checked_add(b as isize)? as usize)
+        })
+    }
+
+    fn output_layout_with_checker(
+        source: &[u8],
+        preferred_quote: Quote,
+        reserved_len: usize,
+        length_add: impl Fn(usize, usize) -> Option<usize>,
+    ) -> EscapeLayout {
+        let mut out_len = reserved_len;
+        let mut single_count = 0;
+        let mut double_count = 0;
+
+        for ch in source.iter() {
+            let incr = match ch {
+                b'\'' => {
+                    single_count += 1;
+                    1
+                }
+                b'"' => {
+                    double_count += 1;
+                    1
+                }
+                c => Self::escaped_char_len(*c),
+            };
+            let Some(new_len) = length_add(out_len, incr) else {
+                #[cold]
+                fn stop(single_count: usize, double_count: usize, preferred_quote: Quote) -> EscapeLayout {
+                    EscapeLayout { quote: choose_quote(single_count, double_count, preferred_quote).0, len: None }
+                }
+                return stop(single_count, double_count, preferred_quote);
+            };
+            out_len = new_len;
+        }
+
+        let (quote, num_escaped_quotes) = choose_quote(single_count, double_count, preferred_quote);
+        // we'll be adding backslashes in front of the existing inner quotes
+        let Some(out_len) = length_add(out_len, num_escaped_quotes) else {
+            return EscapeLayout { quote, len: None };
+        };
+
+        EscapeLayout {
+            quote,
+            len: Some(out_len - reserved_len),
+        }
+    }
+
+    fn escaped_char_len(ch: u8) -> usize {
+        match ch {
+            b'\\' | b'\t' | b'\r' | b'\n' => 2,
+            0x20..=0x7e => 1,
+            _ => 4, // \xHH
+        }
+    }
+
+    fn write_char(ch: u8, quote: Quote, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        match ch {
+            b'\t' => formatter.write_str("\\t"),
+            b'\n' => formatter.write_str("\\n"),
+            b'\r' => formatter.write_str("\\r"),
+            0x20..=0x7e => {
+                // printable ascii range
+                if ch == quote.to_byte() || ch == b'\\' {
+                    formatter.write_char('\\')?;
+                }
+                formatter.write_char(ch as char)
+            }
+            ch => write!(formatter, "\\x{ch:02x}"),
+        }
+    }
+}
+
+impl<'a> Escape for AsciiEscape<'a> {
+    fn source_len(&self) -> usize {
+        self.source.len()
+    }
+
+    fn layout(&self) -> &EscapeLayout {
+        &self.layout
+    }
+
+    fn write_source(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        formatter.write_str(unsafe {
+            // SAFETY: this function must be called only when source is printable ascii characters
+            std::str::from_utf8_unchecked(self.source)
+        })
+    }
+
+    #[cold]
+    fn write_body_slow(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        for ch in self.source.iter() {
+            Self::write_char(*ch, self.layout().quote, formatter)?;
+        }
+        Ok(())
+    }
+}
+
+pub struct BytesRepr<'r, 'a>(&'r AsciiEscape<'a>);
+
+impl BytesRepr<'_, '_> {
+    pub fn write(&self, formatter: &mut impl std::fmt::Write) -> std::fmt::Result {
+        let quote = self.0.layout().quote.to_char();
+        formatter.write_char('b')?;
+        formatter.write_char(quote)?;
+        self.0.write_body(formatter)?;
+        formatter.write_char(quote)
+    }
+
+    pub fn to_string(&self) -> Option<String> {
+        let mut s = String::with_capacity(self.0.layout().len?);
+        self.write(&mut s).unwrap();
+        Some(s)
+    }
+}
+
+impl std::fmt::Display for BytesRepr<'_, '_> {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.write(formatter)
+    }
+}
diff --git a/literal/src/float.rs b/literal/src/float.rs
new file mode 100644
index 00000000..5c14fcbc
--- /dev/null
+++ b/literal/src/float.rs
@@ -0,0 +1,310 @@
+use crate::format::Case;
+use num_traits::{Float, Zero};
+use std::f64;
+
+pub fn parse_str(literal: &str) -> Option<f64> {
+    parse_inner(literal.trim().as_bytes())
+}
+
+pub fn parse_bytes(literal: &[u8]) -> Option<f64> {
+    parse_inner(trim_slice(literal, |b| b.is_ascii_whitespace()))
+}
+
+fn trim_slice<T>(v: &[T], mut trim: impl FnMut(&T) -> bool) -> &[T] {
+    let mut it = v.iter();
+    // it.take_while_ref(&mut trim).for_each(drop);
+    // hmm.. `&mut slice::Iter<_>` is not `Clone`
+    // it.by_ref().rev().take_while_ref(&mut trim).for_each(drop);
+    while it.clone().next().map_or(false, &mut trim) {
+        it.next();
+    }
+    while it.clone().next_back().map_or(false, &mut trim) {
+        it.next_back();
+    }
+    it.as_slice()
+}
+
+fn parse_inner(literal: &[u8]) -> Option<f64> {
+    use lexical_parse_float::{
+        format::PYTHON3_LITERAL, FromLexicalWithOptions, NumberFormatBuilder, Options,
+    };
+    // lexical-core's format::PYTHON_STRING is inaccurate
+    const PYTHON_STRING: u128 = NumberFormatBuilder::rebuild(PYTHON3_LITERAL)
+        .no_special(false)
+        .build();
+    f64::from_lexical_with_options::<PYTHON_STRING>(literal, &Options::new()).ok()
+}
+
+pub fn is_integer(v: f64) -> bool {
+    (v - v.round()).abs() < f64::EPSILON
+}
+
+fn format_nan(case: Case) -> String {
+    let nan = match case {
+        Case::Lower => "nan",
+        Case::Upper => "NAN",
+    };
+
+    nan.to_string()
+}
+
+fn format_inf(case: Case) -> String {
+    let inf = match case {
+        Case::Lower => "inf",
+        Case::Upper => "INF",
+    };
+
+    inf.to_string()
+}
+
+pub fn decimal_point_or_empty(precision: usize, alternate_form: bool) -> &'static str {
+    match (precision, alternate_form) {
+        (0, true) => ".",
+        _ => "",
+    }
+}
+
+pub fn format_fixed(precision: usize, magnitude: f64, case: Case, alternate_form: bool) -> String {
+    match magnitude {
+        magnitude if magnitude.is_finite() => {
+            let point = decimal_point_or_empty(precision, alternate_form);
+            format!("{magnitude:.precision$}{point}")
+        }
+        magnitude if magnitude.is_nan() => format_nan(case),
+        magnitude if magnitude.is_infinite() => format_inf(case),
+        _ => "".to_string(),
+    }
+}
+
+// Formats floats into Python style exponent notation, by first formatting in Rust style
+// exponent notation (`1.0000e0`), then convert to Python style (`1.0000e+00`).
+pub fn format_exponent(
+    precision: usize,
+    magnitude: f64,
+    case: Case,
+    alternate_form: bool,
+) -> String {
+    match magnitude {
+        magnitude if magnitude.is_finite() => {
+            let r_exp = format!("{magnitude:.precision$e}");
+            let mut parts = r_exp.splitn(2, 'e');
+            let base = parts.next().unwrap();
+            let exponent = parts.next().unwrap().parse::<i64>().unwrap();
+            let e = match case {
+                Case::Lower => 'e',
+                Case::Upper => 'E',
+            };
+            let point = decimal_point_or_empty(precision, alternate_form);
+            format!("{base}{point}{e}{exponent:+#03}")
+        }
+        magnitude if magnitude.is_nan() => format_nan(case),
+        magnitude if magnitude.is_infinite() => format_inf(case),
+        _ => "".to_string(),
+    }
+}
+
+/// If s represents a floating point value, trailing zeros and a possibly trailing
+/// decimal point will be removed.
+/// This function does NOT work with decimal commas.
+fn maybe_remove_trailing_redundant_chars(s: String, alternate_form: bool) -> String {
+    if !alternate_form && s.contains('.') {
+        // only truncate floating point values when not in alternate form
+        let s = remove_trailing_zeros(s);
+        remove_trailing_decimal_point(s)
+    } else {
+        s
+    }
+}
+
+fn remove_trailing_zeros(s: String) -> String {
+    let mut s = s;
+    while s.ends_with('0') {
+        s.pop();
+    }
+    s
+}
+
+fn remove_trailing_decimal_point(s: String) -> String {
+    let mut s = s;
+    if s.ends_with('.') {
+        s.pop();
+    }
+    s
+}
+
+pub fn format_general(
+    precision: usize,
+    magnitude: f64,
+    case: Case,
+    alternate_form: bool,
+    always_shows_fract: bool,
+) -> String {
+    match magnitude {
+        magnitude if magnitude.is_finite() => {
+            let r_exp = format!("{:.*e}", precision.saturating_sub(1), magnitude);
+            let mut parts = r_exp.splitn(2, 'e');
+            let base = parts.next().unwrap();
+            let exponent = parts.next().unwrap().parse::<i64>().unwrap();
+            if exponent < -4 || exponent + (always_shows_fract as i64) >= (precision as i64) {
+                let e = match case {
+                    Case::Lower => 'e',
+                    Case::Upper => 'E',
+                };
+                let magnitude = format!("{:.*}", precision + 1, base);
+                let base = maybe_remove_trailing_redundant_chars(magnitude, alternate_form);
+                let point = decimal_point_or_empty(precision.saturating_sub(1), alternate_form);
+                format!("{base}{point}{e}{exponent:+#03}")
+            } else {
+                let precision = ((precision as i64) - 1 - exponent) as usize;
+                let magnitude = format!("{magnitude:.precision$}");
+                let base = maybe_remove_trailing_redundant_chars(magnitude, alternate_form);
+                let point = decimal_point_or_empty(precision, alternate_form);
+                format!("{base}{point}")
+            }
+        }
+        magnitude if magnitude.is_nan() => format_nan(case),
+        magnitude if magnitude.is_infinite() => format_inf(case),
+        _ => "".to_string(),
+    }
+}
+
+// TODO: rewrite using format_general
+pub fn to_string(value: f64) -> String {
+    let lit = format!("{value:e}");
+    if let Some(position) = lit.find('e') {
+        let significand = &lit[..position];
+        let exponent = &lit[position + 1..];
+        let exponent = exponent.parse::<i32>().unwrap();
+        if exponent < 16 && exponent > -5 {
+            if is_integer(value) {
+                format!("{value:.1?}")
+            } else {
+                value.to_string()
+            }
+        } else {
+            format!("{significand}e{exponent:+#03}")
+        }
+    } else {
+        let mut s = value.to_string();
+        s.make_ascii_lowercase();
+        s
+    }
+}
+
+pub fn from_hex(s: &str) -> Option<f64> {
+    if let Ok(f) = hexf_parse::parse_hexf64(s, false) {
+        return Some(f);
+    }
+    match s.to_ascii_lowercase().as_str() {
+        "nan" | "+nan" | "-nan" => Some(f64::NAN),
+        "inf" | "infinity" | "+inf" | "+infinity" => Some(f64::INFINITY),
+        "-inf" | "-infinity" => Some(f64::NEG_INFINITY),
+        value => {
+            let mut hex = String::with_capacity(value.len());
+            let has_0x = value.contains("0x");
+            let has_p = value.contains('p');
+            let has_dot = value.contains('.');
+            let mut start = 0;
+
+            if !has_0x && value.starts_with('-') {
+                hex.push_str("-0x");
+                start += 1;
+            } else if !has_0x {
+                hex.push_str("0x");
+                if value.starts_with('+') {
+                    start += 1;
+                }
+            }
+
+            for (index, ch) in value.chars().enumerate() {
+                if ch == 'p' {
+                    if has_dot {
+                        hex.push('p');
+                    } else {
+                        hex.push_str(".p");
+                    }
+                } else if index >= start {
+                    hex.push(ch);
+                }
+            }
+
+            if !has_p && has_dot {
+                hex.push_str("p0");
+            } else if !has_p && !has_dot {
+                hex.push_str(".p0")
+            }
+
+            hexf_parse::parse_hexf64(hex.as_str(), false).ok()
+        }
+    }
+}
+
+pub fn to_hex(value: f64) -> String {
+    let (mantissa, exponent, sign) = value.integer_decode();
+    let sign_fmt = if sign < 0 { "-" } else { "" };
+    match value {
+        value if value.is_zero() => format!("{sign_fmt}0x0.0p+0"),
+        value if value.is_infinite() => format!("{sign_fmt}inf"),
+        value if value.is_nan() => "nan".to_owned(),
+        _ => {
+            const BITS: i16 = 52;
+            const FRACT_MASK: u64 = 0xf_ffff_ffff_ffff;
+            format!(
+                "{}{:#x}.{:013x}p{:+}",
+                sign_fmt,
+                mantissa >> BITS,
+                mantissa & FRACT_MASK,
+                exponent + BITS
+            )
+        }
+    }
+}
+
+#[test]
+fn test_to_hex() {
+    use rand::Rng;
+    for _ in 0..20000 {
+        let bytes = rand::thread_rng().gen::<[u64; 1]>();
+        let f = f64::from_bits(bytes[0]);
+        if !f.is_finite() {
+            continue;
+        }
+        let hex = to_hex(f);
+        // println!("{} -> {}", f, hex);
+        let roundtrip = hexf_parse::parse_hexf64(&hex, false).unwrap();
+        // println!("  -> {}", roundtrip);
+        assert!(f == roundtrip, "{} {} {}", f, hex, roundtrip);
+    }
+}
+
+#[test]
+fn test_remove_trailing_zeros() {
+    assert!(remove_trailing_zeros(String::from("100")) == *"1");
+    assert!(remove_trailing_zeros(String::from("100.00")) == *"100.");
+
+    // leave leading zeros untouched
+    assert!(remove_trailing_zeros(String::from("001")) == *"001");
+
+    // leave strings untouched if they don't end with 0
+    assert!(remove_trailing_zeros(String::from("101")) == *"101");
+}
+
+#[test]
+fn test_remove_trailing_decimal_point() {
+    assert!(remove_trailing_decimal_point(String::from("100.")) == *"100");
+    assert!(remove_trailing_decimal_point(String::from("1.")) == *"1");
+
+    // leave leading decimal points untouched
+    assert!(remove_trailing_decimal_point(String::from(".5")) == *".5");
+}
+
+#[test]
+fn test_maybe_remove_trailing_redundant_chars() {
+    assert!(maybe_remove_trailing_redundant_chars(String::from("100."), true) == *"100.");
+    assert!(maybe_remove_trailing_redundant_chars(String::from("100."), false) == *"100");
+    assert!(maybe_remove_trailing_redundant_chars(String::from("1."), false) == *"1");
+    assert!(maybe_remove_trailing_redundant_chars(String::from("10.0"), false) == *"10");
+
+    // don't truncate integers
+    assert!(maybe_remove_trailing_redundant_chars(String::from("1000"), false) == *"1000");
+}
diff --git a/literal/src/format.rs b/literal/src/format.rs
new file mode 100644
index 00000000..3d5d9d7b
--- /dev/null
+++ b/literal/src/format.rs
@@ -0,0 +1,5 @@
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum Case {
+    Lower,
+    Upper,
+}
diff --git a/literal/src/lib.rs b/literal/src/lib.rs
new file mode 100644
index 00000000..9b962057
--- /dev/null
+++ b/literal/src/lib.rs
@@ -0,0 +1,4 @@
+pub mod char;
+pub mod escape;
+pub mod float;
+pub mod format;

From 7159de863c05df75a487306cc4baacd290019158 Mon Sep 17 00:00:00 2001
From: Jeong YunWon <jeong@youknowone.org>
Date: Fri, 5 May 2023 23:49:34 +0900
Subject: [PATCH 459/459] Set up workspace - Forked from

git@github.com:RustPython/RustPython.git ff5076b12c075b3e87c0ac2971e390b4a209d14f
---
 .gitignore | 16 +++++++++++++
 Cargo.toml | 69 ++++++++++++++++++++++++++++++++++++++++++++++++------
 LICENSE    | 21 +++++++++++++++++
 3 files changed, 99 insertions(+), 7 deletions(-)
 create mode 100644 .gitignore
 create mode 100644 LICENSE

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..5f2428af
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,16 @@
+Cargo.lock
+
+/target
+/*/target
+**/*.rs.bk
+**/*.bytecode
+__pycache__
+**/*.pytest_cache
+.*sw*
+.vscode
+.idea/
+
+flame-graph.html
+flame.txt
+flamescope.json
+
diff --git a/Cargo.toml b/Cargo.toml
index ff6ea775..5da28823 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,11 +1,66 @@
-[package]
-name = "rustpython-compiler"
+[workspace.package]
 version = "0.2.0"
-description = "A usability wrapper around rustpython-parser and rustpython-compiler-core"
 authors = ["RustPython Team"]
 edition = "2021"
+rust-version = "1.67.1"
+description = "Python parser and its dependencies."
+repository = "https://github.com/RustPython/Parser"
+license = "MIT"
+include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
 
-[dependencies]
-rustpython-compiler-core = { path = "core" }
-rustpython-codegen = { path = "codegen" }
-rustpython-parser = { path = "parser" }
+[workspace]
+resolver = "2"
+members = [
+    "ast", "core", "literal", "parser",
+]
+
+[workspace.dependencies]
+ahash = "0.7.6"
+anyhow = "1.0.45"
+ascii = "1.0"
+atty = "0.2.14"
+bincode = "1.3.3"
+bitflags = "1.3.2"
+bstr = "0.2.17"
+cfg-if = "1.0"
+chrono = "0.4.19"
+crossbeam-utils = "0.8.9"
+flame = "0.2.2"
+glob = "0.3"
+hex = "0.4.3"
+indexmap = "1.8.1"
+insta = "1.14.0"
+itertools = "0.10.3"
+libc = "0.2.133"
+log = "0.4.16"
+nix = "0.26"
+num-complex = "0.4.0"
+num-bigint = "0.4.3"
+num-integer = "0.1.44"
+num-rational = "0.4.0"
+num-traits = "0.2"
+num_enum = "0.5.7"
+once_cell = "1.13"
+parking_lot = "0.12"
+paste = "1.0.7"
+rand = "0.8.5"
+rustyline = "11"
+serde = "1.0"
+schannel = "0.1.19"
+static_assertions = "1.1"
+syn = "1.0.91"
+thiserror = "1.0"
+thread_local = "1.1.4"
+unicode_names2 = { version = "0.6.0", git = "https://github.com/youknowone/unicode_names2.git", rev = "4ce16aa85cbcdd9cc830410f1a72ef9a235f2fde" }
+widestring = "0.5.1"
+
+[profile.dev.package."*"]
+opt-level = 3
+
+[profile.bench]
+lto = "thin"
+codegen-units = 1
+opt-level = 3
+
+[profile.release]
+lto = "thin"
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 00000000..7213274e
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 RustPython Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.