improve heredocs handling. maintain a common string_stack for tstring…

…s and heredocs, make better use of given events (still not optimal, ripper has some oddities)

lib/ripper/ruby_builder.rb

@@ -44,7 +44,7 @@ def build(src, filename = nil)
     include Lexer, Statements, Const, Method, Call, Block, Args, Assignment, Operator,
             If, Case, For, While, Identifier, Literal, String, Symbol, Array, Hash
 
-    attr_reader :src, :filename, :stack, :tstring_stack, :trailing_whitespace
+    attr_reader :src, :filename, :stack, :string_stack, :trailing_whitespace
 
     def initialize(src, filename = nil, lineno = nil)
       @src = src ||= filename && File.read(filename) || ''
@@ -53,7 +53,7 @@ def initialize(src, filename = nil, lineno = nil)
       @filename = filename
       @stack = []
       @stack = Stack.new
-      @tstring_stack = []
+      @string_stack = []
 
       super
     end
@@ -71,7 +71,6 @@ def prolog
       def push(sexp = nil)
         token = Token.new(sexp[0], sexp[1], position) if sexp.is_a?(::Array)
         stack.push(token)
-        end_heredoc(token)
         token
       end
 

lib/ripper/ruby_builder/buffer.rb

@@ -4,7 +4,7 @@ class Buffer < Array
       def flush(options = {})
         self.dup.tap { self.clear }
       end
-      
+
       def aggregate(token)
         if token.nil?
           false

lib/ripper/ruby_builder/events/array.rb

@@ -11,15 +11,15 @@ def on_words_new(*args)
         rdelim = pop_token(:@words_end)
         ldelim = pop_token(:@words_beg)
         words = Ruby::Array.new(nil, ldelim, rdelim)
-        tstring_stack << words
+        string_stack << words
         words
       end
 
       def on_qwords_new(*args)
         rdelim = pop_token(:@words_end)
         ldelim = pop_token(:@qwords_beg)
         words = Ruby::Array.new(nil, ldelim, rdelim)
-        tstring_stack << words # there's no @qwords_end event, so we hook into @tstring_end
+        string_stack << words # there's no @qwords_end event, so we hook into @tstring_end
         words
       end
 
@@ -36,9 +36,9 @@ def on_qwords_add(array, arg)
       end
 
       def on_words_end(rdelim = nil)
-        array = tstring_stack.pop
-        array.rdelim ||= pop_token(:@tstring_end, :@words_sep)
-        array
+        words = string_stack.pop
+        words.rdelim ||= pop_token(:@tstring_end, :@words_sep)
+        words
       end
 
       def on_aref(target, args)
@@ -60,8 +60,8 @@ def on_aref_field(target, args)
         WORD_DELIMITER_MAP = { '(' => ')', '[' => ']', '{' => '}' }
 
         def closes_words?(token)
-          return false unless tstring_stack.last.try(:ldelim)
-          return false unless tstring_stack.last.ldelim.token =~ /^%w\s*([^\s]*)/i
+          return false unless string_stack.last.try(:ldelim)
+          return false unless string_stack.last.ldelim.token =~ /^%w\s*([^\s]*)/i
           (WORD_DELIMITER_MAP[$1] || $1) == token.gsub(/[%w\s]/i, '')
         end
     end

lib/ripper/ruby_builder/events/lexer.rb

@@ -2,23 +2,49 @@ class Ripper
   class RubyBuilder < Ripper::SexpBuilder
     module Lexer
       # unimplemented = :tlambda, :tlambeg
-      
+
       def on_parse_error(msg)
         raise ParseError.new("#{filename}:#{position.row + 1}: #{msg}")
       end
-
+
+      def on_ignored_nl(*args)
+        token = push(super)
+        on_heredoc_literal if end_heredoc?(token)
+        token
+      end
+
+      def on_nl(*args)
+        token = push(super)
+        on_heredoc_literal if end_heredoc?(token)
+        token
+      end
+
+      def on_comment(*args)
+        token = push(super)
+        on_heredoc_literal if end_heredoc?(token)
+        token
+      end
+
       def on_sp(*args)
         push(super)
       end
 
-      def on_nl(*args)
+      def on_semicolon(*args)
         push(super)
       end
 
-      def on_ignored_nl(*args)
+      def on_period(*args)
+        push(super)
+      end
+
+      def on_comma(*args)
         push(super)
       end
-
+
+      def on_backtick(*args)
+        push(super)
+      end
+
       def on_symbeg(*args)
         push(super)
       end
@@ -29,7 +55,8 @@ def on_tstring_beg(*args)
 
       def on_tstring_end(token)
         push(super)
-        on_words_sep(token) && on_words_end(token) if closes_words?(token)
+        # on_words_sep(token) && on_words_end(token) if closes_words?(token) # simulating words events
+        on_words_end(token) if closes_words?(token) # simulating words events
       end
 
       def on_qwords_beg(*args)
@@ -39,7 +66,7 @@ def on_qwords_beg(*args)
       def on_words_beg(*args)
         push(super)
       end
-      
+
       def on_words_sep(token)
         token.each_char do |token|
           case token
@@ -49,22 +76,23 @@ def on_words_sep(token)
             push([:@sp, token, position])
           else
             push([:@words_end, token, position])
+            on_words_end(token) if closes_words?(token)
           end
         end
       end
-      
+
       def on_embexpr_beg(*args)
         push(super)
       end
-      
+
       def on_embexpr_end(*args)
         push(super)
       end
-      
+
       def on_regexp_beg(*args)
         push(super)
       end
-      
+
       def on_regexp_end(*args)
         push(super)
       end
@@ -88,7 +116,7 @@ def on_rbracket(*args)
       def on_lbrace(*args)
         push(super)
       end
-      
+
       def on_rbrace(*args)
         push(super)
       end
@@ -97,42 +125,22 @@ def on_op(*args)
         push(super)
       end
 
-      def on_comma(*args)
-        push(super)
-      end
-
-      def on_backtick(*args)
-        push(super)
-      end
-
-      def on_period(*args)
-        push(super)
-      end
-
-      def on_semicolon(*args)
-        push(super)
-      end
-
-      def on_comment(*args)
-        push(super)
-      end
-
       def on_embdoc(doc)
         push([:@comment, doc, position])
       end
-      
+
       def on_embdoc_beg(doc)
         push([:@comment, doc, position])
       end
-      
+
       def on_embdoc_end(doc)
         push([:@comment, doc, position])
       end
-      
+
       def on_embvar(*args)
         push(super)
       end
-      
+
       def on___end__(*args)
         # TODO
         super

lib/ripper/ruby_builder/events/statements.rb

@@ -21,6 +21,7 @@ def on_paren(node)
       end
 
       def on_stmts_add(target, statement)
+        on_words_end if statement.is_a?(Ruby::Array) && !string_stack.empty?
         target.elements << statement if statement
         target
       end

lib/ripper/ruby_builder/events/string.rb

@@ -6,25 +6,34 @@ def on_string_concat(*strings)
       end
 
       def on_string_literal(string)
-        string.rdelim = pop_token(:@tstring_end) unless string.is_a?(Ruby::HeredocBegin)
+        string_stack.pop if string == string_stack.last
+        string.rdelim = pop_token(:@tstring_end) if string.respond_to?(:rdelim)
         string
       end
 
       def on_xstring_literal(string)
+        string_stack.pop if string == string_stack.last # untested? xstring inside of heredoc or regular string
         string.rdelim = pop_token(:@tstring_end)
         string
       end
 
       def on_regexp_literal(string, rdelim)
+        string_stack.pop if string == string_stack.last # untested? regexp inside of heredoc or regular string
         string.rdelim = pop_token(:@regexp_end)
         string
       end
 
+      def on_heredoc_literal(*args)
+        string_stack.each { |heredoc| push([:@heredoc, heredoc]) }
+        string_stack.clear
+        @heredoc_pos = nil
+      end
+
       def on_string_add(string, content)
-        if string.is_a?(Ruby::HeredocBegin)
-          heredocs.last << content # TODO doesn't work when content spans multiple lines, or does it?
+        if heredoc? && content
+          string_stack.last << content
         elsif string && content
-          string << content 
+          string << content
         end
         string
       end
@@ -48,22 +57,31 @@ def on_string_content(*args)
         if ldelim = pop_token(:@heredoc_beg)
           @heredoc_beg = Ruby::HeredocBegin.new(ldelim.token, ldelim.position, ldelim.prolog)
         else
-          tstring_stack << Ruby::String.new(pop_token(:@tstring_beg))
-          tstring_stack.last
+          string_stack << Ruby::String.new(pop_token(:@tstring_beg))
+          string_stack.last
         end
       end
 
       def on_tstring_content(token)
         content = Ruby::StringContent.new(token, position, prolog)
-        content
+        if heredoc?
+          string_stack.last << content
+          nil
+        else
+          content
+        end
+      end
+
+      def on_word_new
+        Ruby::String.new
       end
 
       def on_xstring_new(*args)
         ldelim = pop(:@symbeg, :@backtick, :@regexp_beg, :max => 1, :pass => true).first
-        tstring_stack << build_xstring(ldelim)
-        tstring_stack.last
+        string_stack << build_xstring(ldelim)
+        string_stack.last
       end
-      
+
       def build_xstring(token)
         case token.type
         when :@symbeg
@@ -75,10 +93,6 @@ def build_xstring(token)
         end
       end
 
-      def on_word_new
-        Ruby::String.new
-      end
-
       def on_string_dvar(variable)
         variable = Ruby::DelimitedVariable.new(variable)
         ldelim = pop_token(:@embvar)
@@ -88,24 +102,13 @@ def on_string_dvar(variable)
 
       def on_heredoc_beg(*args)
         token = push(super)
-        heredocs << Ruby::Heredoc.new
+        string_stack << Ruby::Heredoc.new
         heredoc_pos(position.row + 1, 0) unless heredoc_pos
       end
 
       def on_heredoc_end(token)
-        if pos = heredocs.last.position # TODO position calculation, move to position
-          lines = heredocs.last.to_ruby.split("\n")
-          row = pos.row + lines.size - 1
-          col = lines.last.length
-          heredoc_pos(row, col)
-        end
-
-        heredocs.last.rdelim = Ruby::Token.new(token, position)
-        # should be able to add the string in on_string_add instead, no?
-        content = Ruby::StringContent.new(extract_src(heredoc_pos, heredocs.last.rdelim.position), heredoc_pos)
-
-        heredoc_pos(heredocs.last.rdelim.position.row + 1, 0)
-        content
+        string_stack.last.rdelim = Ruby::Token.new(token, position)
+        nil
       end
 
       protected
@@ -115,28 +118,24 @@ def heredocs
         end
 
         def heredoc?
-          !heredocs.empty?
+          string_stack.last.is_a?(Ruby::Heredoc)
         end
-        
+
         def heredoc_pos(*pos)
           pos.empty? ? @heredoc_pos : @heredoc_pos = Ruby::Node::Position.new(*pos)
         end
-
+
+        def end_heredoc?(token)
+          extra_heredoc_stage? && extra_heredoc_char?(token)
+        end
+
         def extra_heredoc_stage?
-          heredoc? && heredocs.last.rdelim
+          heredoc? && !!string_stack.last.rdelim
         end
-        
+
         def extra_heredoc_char?(token)
           token && (token.newline? || token.comment?)
         end
-
-        def end_heredoc(token)
-          if extra_heredoc_stage? && extra_heredoc_char?(token)
-            heredocs.each { |heredoc| push([:@heredoc, heredoc]) }
-            @heredoc.clear
-            @heredoc_pos = nil
-          end
-        end
     end
   end
-end
+end