Move Ripper parser code into SourceParser and hook up handlers. Move …

…old parser into Legacy namespace. Still need to move handlers and rewrite handlers for new ripper AST

benchmarks/ripper_parser.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require 'benchmark'
+require File.dirname(__FILE__) + '/../lib/yard'
+require 'yard/parser/ruby/legacy/ruby_lex'
+
+class YARD::Parser::SourceParser
+  def top_level_parse(statements) statements end
+end
+
+$files_yard = Dir[File.dirname(__FILE__) + '/../lib/**/*.rb'].map {|f| File.read(f) }
+$files_rip = Dir[File.dirname(__FILE__) + '/../lib/**/*.rb'].map {|f| [File.read(f), f] }
+
+TIMES = 2
+Benchmark.bmbm do |x|
+  x.report("yard-parser  ") { TIMES.times { $files_yard.each {|f| YARD::Parser::Ruby::Legacy::StatementList.new(f) } } }
+  x.report("rip-parser") { TIMES.times { $files_rip.each {|f| YARD::Parser::Ruby::RubyParser.parse(*f) } } }
+  #x.report("old-ripper-parser") { $files.each {|f| OldRipper::RipperSexp.parse(f) } }
+end

lib/yard/autoload.rb

@@ -64,30 +64,57 @@ module Helpers
   end
 
   module Handlers
-    autoload :AliasHandler,         'yard/handlers/alias_handler'
-    autoload :AttributeHandler,     'yard/handlers/attribute_handler'
-    autoload :Base,                 'yard/handlers/base'
-    autoload :ClassHandler,         'yard/handlers/class_handler'
-    autoload :ClassVariableHandler, 'yard/handlers/class_variable_handler'
-    autoload :ConstantHandler,      'yard/handlers/constant_handler'
-    autoload :ExceptionHandler,     'yard/handlers/exception_handler'
-    autoload :MethodHandler,        'yard/handlers/method_handler'
-    autoload :MixinHandler,         'yard/handlers/mixin_handler'
-    autoload :ModuleHandler,        'yard/handlers/module_handler'
-    autoload :VisibilityHandler,    'yard/handlers/visibility_handler'
-    autoload :UndocumentableError,  'yard/handlers/base'
-    autoload :YieldHandler,         'yard/handlers/yield_handler'
+    autoload :Base,                   'yard/handlers/base'
+    autoload :Processor,              'yard/handlers/processor'
+
+    if RUBY18
+      autoload :AliasHandler,         'yard/handlers/alias_handler'
+      autoload :AttributeHandler,     'yard/handlers/attribute_handler'
+      autoload :ClassHandler,         'yard/handlers/class_handler'
+      autoload :ClassVariableHandler, 'yard/handlers/class_variable_handler'
+      autoload :ConstantHandler,      'yard/handlers/constant_handler'
+      autoload :ExceptionHandler,     'yard/handlers/exception_handler'
+      autoload :MethodHandler,        'yard/handlers/method_handler'
+      autoload :MixinHandler,         'yard/handlers/mixin_handler'
+      autoload :ModuleHandler,        'yard/handlers/module_handler'
+      autoload :Processor,            'yard/handlers/processor'
+      autoload :VisibilityHandler,    'yard/handlers/visibility_handler'
+      autoload :UndocumentableError,  'yard/handlers/base'
+      autoload :YieldHandler,         'yard/handlers/yield_handler'
+    end
+
+    module Ruby
+      if RUBY18
+        module Legacy
+          autoload :Base,             'yard/handlers/ruby/legacy/base'
+          autoload :Processor,        'yard/handlers/ruby/legacy/processor'
+        end
+      else
+        autoload :Base,               'yard/handlers/ruby/base'
+        autoload :Processor,          'yard/handlers/ruby/processor'
+      end
+    end
   end
 
   module Parser
-    module RubyToken
-      require 'yard/parser/ruby_lex' # Too much to include manually
+    module Ruby
+      module Legacy
+        autoload :Statement,      'yard/parser/ruby/legacy/statement'
+        autoload :StatementList,  'yard/parser/ruby/legacy/statement_list'
+        autoload :TokenList,      'yard/parser/ruby/legacy/token_list'
+      end
+
+      module RubyToken
+        if RUBY18
+          require 'yard/parser/ruby/legacy/ruby_lex' # Too much to include manually
+        end
+      end
+
+      autoload :AstNode,          'yard/parser/ruby/ast_node'
+      autoload :RubyParser,       'yard/parser/ruby/ruby_parser'
     end
-
-    autoload :SourceParser,   'yard/parser/source_parser'
-    autoload :Statement,      'yard/parser/statement'
-    autoload :StatementList,  'yard/parser/statement_list'
-    autoload :TokenList,      'yard/parser/token_list'
+
+    autoload :SourceParser,       'yard/parser/source_parser'
   end
 
   module Rake

lib/yard/code_objects/base.rb

@@ -170,7 +170,7 @@ def method_missing(meth, *args, &block)
       #   the +Parser::Statement+ holding the source code or the raw source 
       #   as a +String+ for the definition of the code object only (not the block)
       def source=(statement)
-        if statement.is_a? Parser::Statement
+        if statement.is_a? Parser::Ruby::Legacy::Statement
           src = statement.tokens.to_s
           blk = statement.block ? statement.block.to_s : ""
           if src =~ /^def\s.*[^\)]$/ && blk[0,1] !~ /\r|\n/

lib/yard/generators/helpers/html_helper.rb

@@ -182,15 +182,15 @@ def url_for_file(filename, anchor = nil)
       end
 
       def html_syntax_highlight(source)
-        tokenlist = Parser::TokenList.new(source)
+        tokenlist = Parser::Ruby::Legacy::TokenList.new(source)
         tokenlist.map do |s| 
           prettyclass = s.class.class_name.sub(/^Tk/, '').downcase
           prettysuper = s.class.superclass.class_name.sub(/^Tk/, '').downcase
 
           case s
-          when Parser::RubyToken::TkWhitespace, Parser::RubyToken::TkUnknownChar
+          when Parser::Ruby::Legacy::RubyToken::TkWhitespace, Parser::Ruby::Legacy::RubyToken::TkUnknownChar
             h s.text
-          when Parser::RubyToken::TkId
+          when Parser::Ruby::Legacy::RubyToken::TkId
             prettyval = h(s.text)
             "<span class='#{prettyval} #{prettyclass} #{prettysuper}'>#{prettyval}</span>"
           else

lib/yard/handlers/alias_handler.rb

@@ -1,4 +1,4 @@
-class YARD::Handlers::AliasHandler < YARD::Handlers::Base
+class YARD::Handlers::AliasHandler < YARD::Handlers::Ruby::Legacy::Base
   handles /\Aalias(_method)?(\s|\()/
 
   def process

lib/yard/handlers/attribute_handler.rb

@@ -1,4 +1,4 @@
-class YARD::Handlers::AttributeHandler < YARD::Handlers::Base
+class YARD::Handlers::AttributeHandler < YARD::Handlers::Ruby::Legacy::Base
   handles /\Aattr(?:_(?:reader|writer|accessor))?(?:\s|\()/
 
   def process