Merge pull request RustPython#1160 from RustPython/ast-module

Add location to expressions.

Author: windelbouwman

compiler/src/compile.rs

@@ -26,7 +26,8 @@
 shift = 3
 def print_node(node, indent=0):
     if isinstance(node, ast.AST):
-        print(' '*indent, "NODE", node.__class__.__name__)
+        lineno = 'row={}'.format(node.lineno) if hasattr(node, 'lineno') else ''
+        print(' '*indent, "NODE", node.__class__.__name__, lineno)
         for field in node._fields:
             print(' '*indent,'-', field)
             f = getattr(node, field)
@@ -41,12 +42,25 @@ def print_node(node, indent=0):
 print_node(t)
 
 # print(ast.dump(t))
+flag_names = [
+    'is_referenced',
+    'is_assigned',
+    'is_global',
+    'is_local',
+    'is_parameter',
+    'is_free',
+]
 
 def print_table(table, indent=0):
     print(' '*indent, 'table:', table.get_name())
     print(' '*indent, ' ', 'Syms:')
     for sym in table.get_symbols():
-        print(' '*indent, '  ', sym.get_name(), 'is_referenced=', sym.is_referenced(), 'is_assigned=', sym.is_assigned())
+        flags = []
+        for flag_name in flag_names:
+            func = getattr(sym, flag_name)
+            if func():
+                flags.append(flag_name)
+        print(' '*indent, '   sym:', sym.get_name(), 'flags:', ' '.join(flags))
     print(' '*indent, ' ', 'Child tables:')
     for child in table.get_children():
         print_table(child, indent=indent+shift)

compiler/src/symboltable.rs

@@ -17,13 +17,13 @@ pub struct Node {
 #[derive(Debug, PartialEq)]
 pub enum Top {
     Program(Program),
-    Statement(Vec<LocatedStatement>),
+    Statement(Vec<Statement>),
     Expression(Expression),
 }
 
 #[derive(Debug, PartialEq)]
 pub struct Program {
-    pub statements: Vec<LocatedStatement>,
+    pub statements: Vec<Statement>,
 }
 
 #[derive(Debug, PartialEq)]
@@ -38,12 +38,12 @@ pub struct Located<T> {
     pub node: T,
 }
 
-pub type LocatedStatement = Located<Statement>;
+pub type Statement = Located<StatementType>;
 
 /// Abstract syntax tree nodes for python statements.
 #[allow(clippy::large_enum_variant)]
 #[derive(Debug, PartialEq)]
-pub enum Statement {
+pub enum StatementType {
     Break,
     Continue,
     Return {
@@ -85,58 +85,58 @@ pub enum Statement {
     },
     If {
         test: Expression,
-        body: Vec<LocatedStatement>,
-        orelse: Option<Vec<LocatedStatement>>,
+        body: Vec<Statement>,
+        orelse: Option<Vec<Statement>>,
     },
     While {
         test: Expression,
-        body: Vec<LocatedStatement>,
-        orelse: Option<Vec<LocatedStatement>>,
+        body: Vec<Statement>,
+        orelse: Option<Vec<Statement>>,
     },
     With {
         items: Vec<WithItem>,
-        body: Vec<LocatedStatement>,
+        body: Vec<Statement>,
     },
     For {
         target: Expression,
         iter: Expression,
-        body: Vec<LocatedStatement>,
-        orelse: Option<Vec<LocatedStatement>>,
+        body: Vec<Statement>,
+        orelse: Option<Vec<Statement>>,
     },
     AsyncFor {
         target: Expression,
         iter: Expression,
-        body: Vec<LocatedStatement>,
-        orelse: Option<Vec<LocatedStatement>>,
+        body: Vec<Statement>,
+        orelse: Option<Vec<Statement>>,
     },
     Raise {
         exception: Option<Expression>,
         cause: Option<Expression>,
     },
     Try {
-        body: Vec<LocatedStatement>,
+        body: Vec<Statement>,
         handlers: Vec<ExceptHandler>,
-        orelse: Option<Vec<LocatedStatement>>,
-        finalbody: Option<Vec<LocatedStatement>>,
+        orelse: Option<Vec<Statement>>,
+        finalbody: Option<Vec<Statement>>,
     },
     ClassDef {
         name: String,
-        body: Vec<LocatedStatement>,
+        body: Vec<Statement>,
         bases: Vec<Expression>,
         keywords: Vec<Keyword>,
         decorator_list: Vec<Expression>,
     },
     FunctionDef {
         name: String,
         args: Parameters,
-        body: Vec<LocatedStatement>,
+        body: Vec<Statement>,
         decorator_list: Vec<Expression>,
         returns: Option<Expression>,
     },
     AsyncFunctionDef {
         name: String,
         args: Parameters,
-        body: Vec<LocatedStatement>,
+        body: Vec<Statement>,
         decorator_list: Vec<Expression>,
         returns: Option<Expression>,
     },
@@ -148,8 +148,10 @@ pub struct WithItem {
     pub optional_vars: Option<Expression>,
 }
 
+pub type Expression = Located<ExpressionType>;
+
 #[derive(Debug, PartialEq)]
-pub enum Expression {
+pub enum ExpressionType {
     BoolOp {
         a: Box<Expression>,
         op: BooleanOperator,
@@ -242,10 +244,10 @@ pub enum Expression {
 impl Expression {
     /// Returns a short name for the node suitable for use in error messages.
     pub fn name(&self) -> &'static str {
-        use self::Expression::*;
+        use self::ExpressionType::*;
         use self::StringGroup::*;
 
-        match self {
+        match &self.node {
             BoolOp { .. } | Binop { .. } | Unop { .. } => "operator",
             Subscript { .. } => "subscript",
             Await { .. } => "await expression",
@@ -305,6 +307,7 @@ pub struct Parameter {
     pub annotation: Option<Box<Expression>>,
 }
 
+#[allow(clippy::large_enum_variant)]
 #[derive(Debug, PartialEq)]
 pub enum ComprehensionKind {
     GeneratorExpression { element: Expression },
@@ -330,7 +333,7 @@ pub struct Keyword {
 pub struct ExceptHandler {
     pub typ: Option<Expression>,
     pub name: Option<String>,
-    pub body: Vec<LocatedStatement>,
+    pub body: Vec<Statement>,
 }
 
 #[derive(Debug, PartialEq)]

crawl_sourcecode.py

@@ -173,9 +173,12 @@ mod tests {
 
     use super::*;
 
-    fn mk_ident(name: &str) -> ast::Expression {
-        ast::Expression::Identifier {
-            name: name.to_owned(),
+    fn mk_ident(name: &str, row: usize, col: usize) -> ast::Expression {
+        ast::Expression {
+            location: ast::Location::new(row, col),
+            node: ast::ExpressionType::Identifier {
+                name: name.to_owned(),
+            },
         }
     }
 
@@ -189,12 +192,12 @@ mod tests {
             Joined {
                 values: vec![
                     FormattedValue {
-                        value: Box::new(mk_ident("a")),
+                        value: Box::new(mk_ident("a", 1, 1)),
                         conversion: None,
                         spec: String::new(),
                     },
                     FormattedValue {
-                        value: Box::new(mk_ident("b")),
+                        value: Box::new(mk_ident("b", 1, 1)),
                         conversion: None,
                         spec: String::new(),
                     },