Add 'yield from' syntax

Author: windelbouwman

parser/src/lexer.rs

@@ -268,14 +268,14 @@ where
         // Skip everything until end of line
         self.next_char();
         loop {
-            self.next_char();
             match self.chr0 {
                 Some('\n') => {
                     return;
                 }
                 Some(_) => {}
                 None => return,
             }
+            self.next_char();
         }
     }
 

tests/snippets/comments.py

@@ -0,0 +1,12 @@
+
+# Snippet to demo comment handling...
+
+
+def foo():
+    a = []
+
+    # This empty comment below manifests a bug:
+    #
+    if len(a) > 2:
+        a.append(2)
+    return a

tests/snippets/generators.py

@@ -25,3 +25,13 @@ def g2(x):
 assert 15 == i.send(10)
 assert 17 == i.send(10)
 
+
+def g3():
+    yield 23
+    yield from make_numbers()
+    yield 44
+
+r = list(g3())
+# print(r)
+assert r == [23, 1, 2, 3, 44]
+

vm/src/bytecode.rs

@@ -130,6 +130,7 @@ pub enum Instruction {
     ForIter,
     ReturnValue,
     YieldValue,
+    YieldFrom,
     SetupLoop {
         start: Label,
         end: Label,

vm/src/compile.rs

@@ -293,7 +293,7 @@ impl Compiler {
                     }
                 }
                 None => {
-                    unimplemented!();
+                    self.emit(Instruction::Raise { argc: 0 });
                 }
             },
             ast::Statement::Try {
@@ -792,6 +792,7 @@ impl Compiler {
                                 value: name.to_string(),
                             });
                         } else {
+                            // This means **kwargs!
                             panic!("name must be set");
                         }
                         self.compile_expression(&keyword.value)?;
@@ -916,8 +917,11 @@ impl Compiler {
             ast::Expression::YieldFrom { value } => {
                 self.mark_generator();
                 self.compile_expression(value)?;
-                self.emit(Instruction::YieldValue);
-                unimplemented!("yield from todo");
+                self.emit(Instruction::GetIter);
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::None,
+                });
+                self.emit(Instruction::YieldFrom);
             }
             ast::Expression::True => {
                 self.emit(Instruction::LoadConst {

vm/src/frame.rs

@@ -337,6 +337,31 @@ impl Frame {
                 let value = self.pop_value();
                 Some(Ok(ExecutionResult::Yield(value)))
             }
+            bytecode::Instruction::YieldFrom => {
+                // Value send into iterator:
+                self.pop_value();
+
+                let top_of_stack = self.last_value();
+                let next_obj = objiter::get_next_object(vm, &top_of_stack);
+
+                match next_obj {
+                    Ok(Some(value)) => {
+                        // Set back program counter:
+                        self.lasti -= 1;
+                        Some(Ok(ExecutionResult::Yield(value)))
+                    }
+                    Ok(None) => {
+                        // Pop iterator from stack:
+                        self.pop_value();
+                        None
+                    }
+                    Err(next_error) => {
+                        // Pop iterator from stack:
+                        self.pop_value();
+                        Some(Err(next_error))
+                    }
+                }
+            }
             bytecode::Instruction::SetupLoop { start, end } => {
                 self.push_block(Block::Loop {
                     start: *start,
@@ -398,35 +423,31 @@ impl Frame {
             bytecode::Instruction::ForIter => {
                 // The top of stack contains the iterator, lets push it forward:
                 let top_of_stack = self.last_value();
-                let next_obj: PyResult = vm.call_method(&top_of_stack, "__next__", vec![]);
+                let next_obj = objiter::get_next_object(vm, &top_of_stack);
 
                 // Check the next object:
                 match next_obj {
-                    Ok(value) => {
+                    Ok(Some(value)) => {
                         self.push_value(value);
                         None
                     }
-                    Err(next_error) => {
-                        // Check if we have stopiteration, or something else:
-                        if objtype::isinstance(
-                            &next_error,
-                            vm.ctx.exceptions.stop_iteration.clone(),
-                        ) {
-                            // Pop iterator from stack:
-                            self.pop_value();
-
-                            // End of for loop
-                            let end_label = if let Block::Loop { start: _, end } = self.last_block()
-                            {
-                                *end
-                            } else {
-                                panic!("Wrong block type")
-                            };
-                            self.jump(end_label);
-                            None
+                    Ok(None) => {
+                        // Pop iterator from stack:
+                        self.pop_value();
+
+                        // End of for loop
+                        let end_label = if let Block::Loop { start: _, end } = self.last_block() {
+                            *end
                         } else {
-                            Some(Err(next_error))
-                        }
+                            panic!("Wrong block type")
+                        };
+                        self.jump(end_label);
+                        None
+                    }
+                    Err(next_error) => {
+                        // Pop iterator from stack:
+                        self.pop_value();
+                        Some(Err(next_error))
                     }
                 }
             }

vm/src/obj/objiter.rs

@@ -43,6 +43,44 @@ pub fn get_iter(vm: &mut VirtualMachine, iter_target: &PyObjectRef) -> PyResult
     Ok(iter_obj)
 }
 
+/*
+ * Helper function to retrieve the next object (or none) from an iterator.
+ */
+pub fn get_next_object(
+    vm: &mut VirtualMachine,
+    iter_obj: &PyObjectRef,
+) -> Result<Option<PyObjectRef>, PyObjectRef> {
+    let next_obj: PyResult = vm.call_method(iter_obj, "__next__", vec![]);
+
+    match next_obj {
+        Ok(value) => Ok(Some(value)),
+        Err(next_error) => {
+            // Check if we have stopiteration, or something else:
+            if objtype::isinstance(&next_error, vm.ctx.exceptions.stop_iteration.clone()) {
+                Ok(None)
+            } else {
+                Err(next_error)
+            }
+        }
+    }
+}
+
+/* Retrieve all elements from an iterator */
+pub fn get_all(
+    vm: &mut VirtualMachine,
+    iter_obj: &PyObjectRef,
+) -> Result<Vec<PyObjectRef>, PyObjectRef> {
+    let mut elements = vec![];
+    loop {
+        let element = get_next_object(vm, iter_obj)?;
+        match element {
+            Some(v) => elements.push(v),
+            None => break,
+        }
+    }
+    Ok(elements)
+}
+
 // Sequence iterator:
 fn iter_new(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
     arg_check!(vm, args, required = [(iter_target, None)]);

vm/src/obj/objlist.rs

@@ -51,15 +51,8 @@ fn list_new(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult {
     let elements = match iterable {
         None => vec![],
         Some(iterable) => {
-            let mut elements = vec![];
             let iterator = objiter::get_iter(vm, iterable)?;
-            loop {
-                match vm.call_method(&iterator, "__next__", vec![]) {
-                    Ok(v) => elements.push(v),
-                    _ => break,
-                }
-            }
-            elements
+            objiter::get_all(vm, &iterator)?
         }
     };