Start new benchmarking framework

benchmark/bin/benchmark

@@ -0,0 +1,124 @@
+#!/usr/bin/env ruby
+
+lib_path = File.expand_path("../../lib", __FILE__)
+
+$:.unshift lib_path
+
+require 'optparse'
+require 'tempfile'
+
+require 'benchmark/suite'
+require 'benchmark/ips'
+
+targets = []
+at_end = false
+
+opt = OptionParser.new do |o|
+  o.on("-t", "--target TARGET", String,
+          "Use TARGET to compare against: r:ruby|r19:ruby19|x:rbx|j:jruby") do |t|
+    case t
+    when 'r', 'ruby'
+      targets << 'ruby'
+    when 'r19', 'ruby19'
+      targets << 'ruby19'
+    when 'x', 'rbx', 'rubinius'
+      targets << 'bin/rbx'
+    when 'j', 'jruby'
+      targets << 'jruby'
+    else
+      targets << t
+    end
+  end
+
+  o.on("-e", "--end", "Report all stats after all suitse have run") do
+    at_end = true
+  end
+end
+
+opt.parse!
+
+if targets.empty?
+  targets << "bin/rbx"
+end
+
+opts = []
+
+if at_end
+  opts << "--quiet"
+end
+
+results = targets.map do |t|
+  tf = Tempfile.new "benchmark"
+  tf.close
+  puts "=== #{t} ===" unless at_end
+  args = ["-I#{lib_path}", "benchmark/lib/benchmark/suite-run.rb"]
+  args += opts
+  args << tf.path
+  args += ARGV
+
+  cmd, *rest = t.split(/\s+/)
+  args.unshift *rest
+
+  system cmd, *args
+
+  tf.open
+
+  [t, Marshal.load(tf.read)]
+end
+
+if at_end
+  results.each do |name, suite|
+    puts "=== #{name} ==="
+    suite.display
+  end
+end
+
+if targets.size > 1
+  compared = Hash.new { |h,k| h[k] = [] }
+
+  results.each do |target, suite|
+    suite.reports.each do |name, reports|
+      reports.each do |rep|
+        compared["#{name}:#{rep.label}"] << [target, rep]
+      end
+    end
+  end
+
+  puts
+
+  compared.each do |name, reports|
+    if reports.size > 1
+      puts "Comparing #{name}:"
+
+      iter = false
+      sorted = reports.sort do |a,b|
+        if a[1].respond_to? :ips
+          iter = true
+          b[1].ips <=> a[1].ips
+        else
+          a[1].runtime <=> b[1].runtime
+        end
+      end
+
+      best_name, best_report = sorted.shift
+
+
+      if iter
+        printf "%20s: %10d i/s\n", best_name, best_report.ips
+      else
+        puts "#{best_name.rjust(20)}: #{best_report.runtime}s"
+      end
+
+      sorted.each do |entry|
+        name, report = entry
+        if iter
+          x = (best_report.ips.to_f / report.ips.to_f)
+          printf "%20s: %10d i/s - %.2fx slower\n", name, report.ips, x
+        else
+          x = "%.2f" % (report.ips.to_f / best_report.ips.to_f)
+          puts "#{name.rjust(20)}: #{report.runtime}s - #{x}x slower"
+        end
+      end
+    end
+  end
+end

benchmark/lib/benchmark/ips.rb

@@ -0,0 +1,220 @@
+require 'benchmark/timing'
+
+module Benchmark
+
+  class IPSReport
+    def initialize(label, us, iters, ips, ips_sd, cycles)
+      @label = label
+      @microseconds = us
+      @iterations = iters
+      @ips = ips
+      @ips_sd = ips_sd
+      @measurement_cycle = cycles
+    end
+
+    attr_reader :label, :microseconds, :iterations, :ips, :ips_sd, :measurement_cycle
+
+    def seconds
+      @microseconds.to_f / 1_000_000.0
+    end
+
+    def stddev_percentage
+      100.0 * (@ips_sd.to_f / @ips.to_f)
+    end
+
+    alias_method :runtime, :seconds
+
+    def body
+      left = "%10d (±%.1f%%) i/s" % [ips, stddev_percentage]
+      left.ljust(20) + (" - %10d in %10.6fs (cycle=%d)" % 
+                          [@iterations, runtime, @measurement_cycle])
+    end
+
+    def header
+      @label.rjust(20)
+    end
+
+    def to_s
+      "#{header} #{body}"
+    end
+
+    def display
+      puts to_s
+    end
+  end
+
+  class IPSJob
+    class Entry
+      def initialize(label, action)
+        @label = label
+
+        if action.kind_of? String
+          compile action
+          @action = self
+          @as_action = true
+        else
+          unless action.respond_to? :call
+            raise ArgumentError, "invalid action, must respond to #call"
+          end
+
+          @action = action
+
+          if action.respond_to? :arity and action.arity > 0
+            @call_loop = true
+          else
+            @call_loop = false
+          end
+
+          @as_action = false
+        end
+      end
+
+      attr_reader :label, :action
+
+      def as_action?
+        @as_action
+      end
+
+      def call_times(times)
+        return @action.call(times) if @call_loop
+
+        act = @action
+
+        i = 0
+        while i < times
+          act.call
+          i += 1
+        end
+      end
+
+      def compile(str)
+        m = (class << self; self; end)
+        code = <<-CODE
+          def call_times(__total);
+            __i = 0
+            while __i < __total
+              #{str};
+              __i += 1
+            end
+          end
+        CODE
+        m.class_eval code
+      end
+    end
+
+    def initialize
+      @list = []
+    end
+
+    #
+    # Registers the given label and block pair in the job list.
+    #
+    def item(label="", str=nil, &blk) # :yield:
+      if blk and str
+        raise ArgmentError, "specify a block and a str, but not both"
+      end
+
+      action = str || blk
+      raise ArgmentError, "no block or string" unless action
+
+      @list.push Entry.new(label, action)
+      self
+    end
+
+    alias_method :report, :item
+
+    # An array of 2-element arrays, consisting of label and block pairs.
+    attr_reader :list
+  end
+
+  def ips(time=5, warmup=2)
+    suite = nil
+
+    sync, STDOUT.sync = STDOUT.sync, true
+
+    if defined? Benchmark::Suite and Suite.current
+      suite = Benchmark::Suite.current
+    end
+
+    job = IPSJob.new
+    yield job
+
+    start = Timing::TimeVal.new
+    cur = Timing::TimeVal.new
+
+    before = Timing::TimeVal.new
+    after = Timing::TimeVal.new
+
+    job.list.each do |item|
+      suite.warming item.label, warmup if suite
+
+      Timing.clean_env
+
+      STDOUT.printf item.label.rjust(20) if !suite or !suite.quiet?
+
+      before.update!
+      start.update!
+      cur.update!
+
+      warmup_iter = 0
+
+      until start.elapsed?(cur, warmup)
+        item.call_times(1)
+        warmup_iter += 1
+        cur.update!
+      end
+
+      after.update!
+
+      warmup_time = before.diff(after)
+
+      # calculate the time to run approx 100ms
+
+      cycles_per_100ms = ((100_000 / warmup_time.to_f) * warmup_iter).to_i
+      cycles_per_100ms = 1 if cycles_per_100ms <= 0
+
+      suite.warmup_stats warmup_time, cycles_per_100ms if suite
+
+      Timing.clean_env
+
+      suite.running item.label, time if suite
+
+      iter = 0
+      start.update!
+      cur.update!
+
+      measurements = []
+
+      until start.elapsed?(cur, time)
+        before.update!
+        item.call_times cycles_per_100ms
+        after.update!
+
+        iter += cycles_per_100ms
+        cur.update!
+
+        measurements << before.diff(after)
+      end
+
+      measured_us = measurements.inject(0) { |a,i| a + i }
+
+      seconds = measured_us.to_f / 1_000_000.0
+
+      all_ips = measurements.map { |i| cycles_per_100ms.to_f / (i.to_f / 1_000_000) }
+
+      avg_ips = Timing.mean(all_ips)
+      sd_ips =  Timing.stddev(all_ips).round
+
+      rep = IPSReport.new(item.label, measured_us, iter, avg_ips, sd_ips, cycles_per_100ms)
+
+      puts " #{rep.body}" if !suite or !suite.quiet?
+
+      suite.add_report rep, caller(1).first if suite
+
+      STDOUT.sync = sync
+    end
+  end
+
+
+  module_function :ips
+end

benchmark/lib/benchmark/suite-run.rb

@@ -0,0 +1,31 @@
+require 'benchmark/suite'
+require 'optparse'
+
+quiet = false
+
+opts = OptionParser.new do |o|
+  o.on "-q", "--quiet" do
+    quiet = true
+  end
+end
+
+opts.parse!
+
+results = ARGV.shift
+
+suite = Benchmark::Suite.create do |s|
+  s.quiet! if quiet
+
+  ARGV.each do |f|
+    if File.directory?(f)
+      more = Dir["#{f}/**/{bench,bm}_*.rb"]
+      more.each do |x|
+        s.run x
+      end
+    else
+      s.run f
+    end
+  end
+end
+
+File.open(results, "w") { |f| f << Marshal.dump(suite) }