stacked heredocs!

HEREDOC

@@ -1,8 +1,3 @@
-# mir scheint, das pattern hier ist, daß der heredoc body immer in der zeile
-# unmittelbar nach dem heredoc_beg token stehen muß. d.h., es sollte möglich
-# sein, heredoc_beg und body zu trennen, nur heredoc_beg im urspünglichen
-# statement zu lassen und den body in stmt_add zu übernehmen.
-
 class String; def bar; end; end; def foo(str) str; end; def bar; end
 
 <<-end

NOTES

@@ -2,9 +2,18 @@
   make it use a FileIO
   move Text, Position etc. to a new module Source
 
-* refactor RubyBuilder events to class methods on Ruby::Node classes
+* refactor RubyBuilder events to class methods on Ruby::Node classes?
 	e.g. Hash.on_paren would wrap into a Statement while ArgList.on_paren would set the delims
 
+
+* Maybe have a prefix and an suffix for each node so that all nodes would be
+  structured like this:
+
+  	[prefix: separator, whitespace][ldelim][nodes][rdelim][suffix]
+
+  Const namespace path operators could be prefixes. Heredocs could be suffixes.
+
+
 sexp("I18n::t()") # =>
 [:program,
  [:stmts_add,

lib/ripper/ruby_builder.rb

@@ -1,6 +1,5 @@
 require 'ripper'
 require 'ruby'
-require 'ruby/node/context'
 require 'ripper/ruby_builder/token'
 require 'ripper/ruby_builder/stack'
 
@@ -65,9 +64,14 @@ def position
         Ruby::Node::Position.new(lineno.to_i - 1, column)
       end
 
+      def prolog
+        Ruby::Prolog.new(stack.buffer.flush)
+      end
+
       def push(sexp = nil)
-        token = Token.new(sexp[0], sexp[1], position) if sexp
-        stack.push(token) unless extra_heredoc_chars(token)
+        token = Token.new(sexp[0], sexp[1], position) if sexp.is_a?(::Array)
+        stack.push(token)
+        end_heredoc(token)
         token
       end
 
@@ -109,19 +113,15 @@ def pop_assignment_operator(options = {})
         pop_token(*ASSIGN_OPERATORS, options)
       end
 
-      def pop_context
-        stack.context.get
-      end
-
       def build_token(token)
-        Ruby::Token.new(token.token, token.position, token.context) if token
+        Ruby::Token.new(token.token, token.position, token.prolog) if token
       end
 
       def build_keyword(token)
         klass = Ruby.const_get(token.token[0].upcase + token.token[1..-1])
-        klass.new(token, token.position, token.context)
+        klass.new(token, token.position, token.prolog)
       rescue NameError
-        Ruby::Keyword.new(token, token.position, token.context)
+        Ruby::Keyword.new(token, token.position, token.prolog)
       end
 
       def extract_src(from, to)

lib/ripper/ruby_builder/buffer.rb

@@ -0,0 +1,27 @@
+class Ripper
+  class RubyBuilder < Ripper::SexpBuilder
+    class Buffer < Array
+      def flush(options = {})
+        self.dup.tap { self.clear }
+      end
+
+      def aggregate(token)
+        if token.nil?
+          false
+        elsif token.whitespace?
+          self << Ruby::Whitespace.new(token.token, token.position)
+          true
+        elsif token.separator?
+          self << Ruby::Token.new(token.token, token.position)
+          true
+        elsif token.heredoc?
+          self << token.token
+          true
+        else
+          token.prolog = Ruby::Prolog.new(flush) unless empty?
+          false
+        end
+      end
+    end
+  end
+end

lib/ripper/ruby_builder/events/block.rb

@@ -70,7 +70,7 @@ def to_assignments(array)
         # gosh, yes. ripper regards empty block param delims (as in in do || ; end) as an operator
         def empty_block_params_delimiters(params)
           op = pop_token(:'@||')
-          ldelim = Ruby::Token.new('|', op.position, op.context)
+          ldelim = Ruby::Token.new('|', op.position, op.prolog)
           rdelim = Ruby::Token.new('|', op.position).tap { |o| o.position.col += 1 }
           [ldelim, rdelim]
         end

lib/ripper/ruby_builder/events/identifier.rb

@@ -4,37 +4,37 @@ module Identifier
       def on_const(token)
         push(super)
         token = pop_token(:@const)
-        Ruby::Const.new(token.token, token.position, token.context)
+        Ruby::Const.new(token.token, token.position, token.prolog)
       end
 
       def on_ident(token)
         push(super)
         token = pop_token(:@ident)
-        Ruby::Identifier.new(token.token, token.position, token.context)
+        Ruby::Identifier.new(token.token, token.position, token.prolog)
       end
 
       def on_cvar(token)
         push(super)
         token = pop_token(:@cvar)
-        Ruby::Variable.new(token.token, token.position, token.context)
+        Ruby::Variable.new(token.token, token.position, token.prolog)
       end
 
       def on_ivar(token)
         push(super)
         token = pop_token(:@ivar)
-        Ruby::Variable.new(token.token, token.position, token.context)
+        Ruby::Variable.new(token.token, token.position, token.prolog)
       end
 
       def on_gvar(token)
         push(super)
         token = pop_token(:@gvar)
-        Ruby::Variable.new(token.token, token.position, token.context)
+        Ruby::Variable.new(token.token, token.position, token.prolog)
       end
 
       def on_backref(arg)
         push(super)
         token = pop_token(:@backref)
-        Ruby::Variable.new(token.token, token.position, token.context)
+        Ruby::Variable.new(token.token, token.position, token.prolog)
       end
     end
   end

lib/ripper/ruby_builder/events/literal.rb

@@ -17,12 +17,12 @@ def on_kw(token)
 
       def on_int(token)
         push
-        Ruby::Integer.new(token, position, pop_context)
+        Ruby::Integer.new(token, position, prolog)
       end
 
       def on_float(token)
         push
-        Ruby::Float.new(token, position, pop_context)
+        Ruby::Float.new(token, position, prolog)
       end
 
       def on_dot2(left, right)
@@ -35,12 +35,12 @@ def on_dot3(left, right)
 
       def on_CHAR(token)
         push
-        Ruby::Char.new(token, position, pop_context)
+        Ruby::Char.new(token, position, prolog)
       end
 
       def on_label(label)
         push
-        Ruby::Label.new(label, position, pop_context)
+        Ruby::Label.new(label, position, prolog)
       end
     end
   end

lib/ripper/ruby_builder/events/statements.rb

@@ -3,7 +3,7 @@ class RubyBuilder < Ripper::SexpBuilder
     module Statements
       def on_program(statements)
         program = statements.to_program(src, filename)
-        program << Ruby::Token.new('', position, pop_context) unless stack.context.empty?
+        program << Ruby::Token.new('', position, prolog) unless stack.buffer.empty?
         program
       end
 
@@ -34,7 +34,7 @@ def on_void_stmt
       end
 
       def on_var_ref(ref)
-        ref.instance_of?(Ruby::Identifier) ? Ruby::Variable.new(ref.token, ref.position, ref.context) : ref
+        ref.instance_of?(Ruby::Identifier) ? Ruby::Variable.new(ref.token, ref.position, ref.prolog) : ref
       end
 
       def on_const_ref(const)

lib/ripper/ruby_builder/events/string.rb

@@ -22,8 +22,7 @@ def on_regexp_literal(string, rdelim)
 
       def on_string_add(string, content)
         if string.is_a?(Ruby::HeredocBegin)
-          heredoc << content # TODO doesn't work when content spans multiple lines, or does it?
-          heredoc_pos(content.position.row, content.position.col + content.length) 
+          heredocs.last << content # TODO doesn't work when content spans multiple lines, or does it?
         elsif string && content
           string << content 
         end
@@ -47,15 +46,15 @@ def on_string_embexpr(expression)
 
       def on_string_content(*args)
         if ldelim = pop_token(:@heredoc_beg)
-          @heredoc_beg = Ruby::HeredocBegin.new(ldelim.token, ldelim.position, ldelim.context)
+          @heredoc_beg = Ruby::HeredocBegin.new(ldelim.token, ldelim.position, ldelim.prolog)
         else
           tstring_stack << Ruby::String.new(pop_token(:@tstring_beg))
           tstring_stack.last
         end
       end
 
       def on_tstring_content(token)
-        content = Ruby::StringContent.new(token, position, pop_context)
+        content = Ruby::StringContent.new(token, position, prolog)
         content
       end
 
@@ -89,51 +88,54 @@ def on_string_dvar(variable)
 
       def on_heredoc_beg(*args)
         token = push(super)
-        heredoc_pos(position.row + 1, 0)
+        heredocs << Ruby::Heredoc.new
+        heredoc_pos(position.row + 1, 0) unless heredoc_pos
       end
 
-      def on_heredoc_end(*args)
-        push(super)
-
-        if pos = heredoc.position # TODO position calculation, move to position
-          lines = heredoc.to_ruby.split("\n")
+      def on_heredoc_end(token)
+        if pos = heredocs.last.position # TODO position calculation, move to position
+          lines = heredocs.last.to_ruby.split("\n")
           row = pos.row + lines.size - 1
           col = lines.last.length
           heredoc_pos(row, col)
         end
 
-        heredoc.rdelim = pop_token(:@heredoc_end)
-        Ruby::StringContent.new(extract_src(heredoc_pos, heredoc.rdelim.position), heredoc_pos)
+        heredocs.last.rdelim = Ruby::Token.new(token, position)
+        # should be able to add the string in on_string_add instead, no?
+        content = Ruby::StringContent.new(extract_src(heredoc_pos, heredocs.last.rdelim.position), heredoc_pos)
+
+        heredoc_pos(heredocs.last.rdelim.position.row + 1, 0)
+        content
       end
 
       protected
 
-        def heredoc
-          @heredoc ||= Ruby::Heredoc.new
+        def heredocs
+          @heredoc ||= []
+        end
+
+        def heredoc?
+          !heredocs.empty?
         end
 
         def heredoc_pos(*pos)
           pos.empty? ? @heredoc_pos : @heredoc_pos = Ruby::Node::Position.new(*pos)
         end
-
-        def extra_heredoc_chars(token)
-          if extra_heredoc_stage? && extra_heredoc_char?(token)
-            token.token += @heredoc.to_ruby # BIG HACK! ... somehow bubble the heredoc up to a more reasonable place
-            clear_heredoc!
-          end
-          false
-        end
 
         def extra_heredoc_stage?
-          @heredoc && heredoc.rdelim
+          heredoc? && heredocs.last.rdelim
         end
 
         def extra_heredoc_char?(token)
           token && (token.newline? || token.comment?)
         end
 
-        def clear_heredoc!
-          @heredoc_pos = @heredoc_beg = @heredoc = nil
+        def end_heredoc(token)
+          if extra_heredoc_stage? && extra_heredoc_char?(token)
+            heredocs.each { |heredoc| push([:@heredoc, heredoc]) }
+            @heredoc.clear
+            @heredoc_beg = nil
+          end
         end
     end
   end

lib/ripper/ruby_builder/stack.rb

@@ -1,19 +1,19 @@
 require 'core_ext/hash/delete_at'
 require 'ripper/ruby_builder/queue'
-require 'ripper/ruby_builder/context'
+require 'ripper/ruby_builder/buffer'
 
 class Ripper
   class RubyBuilder < Ripper::SexpBuilder
     class Stack < ::Array
-      attr_reader :queue, :context
+      attr_reader :queue, :buffer
 
       def initialize
         @queue = Queue.new
-        @context = Context.new
+        @buffer = Buffer.new
       end
 
       def push(token)
-        return if context.aggregate(token)
+        return token if buffer.aggregate(token)
         tokens = queue << token
         tokens.each do |token|
           self << token