Getting source + tying groups + dependencies together

Gemfile

@@ -2,6 +2,7 @@ source :rubygems
 source "http://gems.github.com"
 
 gem 'bundler', '~>1.2.1'
-gem 'octokit', '=1.22.0'
 gem 'gemnasium-parser', '=0.1.9'
-gem 'xlsx_writer', '=0.4.0'
+gem 'gems', '~>0.7.0'
+gem 'octokit', '~>1.22.0'
+gem 'xlsx_writer', '~>0.4.0'

Gemfile.lock

@@ -12,6 +12,7 @@ GEM
       faraday (>= 0.7.4, < 0.9)
     fast_xs (0.8.0)
     gemnasium-parser (0.1.9)
+    gems (0.7.0)
     hashie (1.2.0)
     i18n (0.6.1)
     multi_json (1.5.0)
@@ -36,5 +37,6 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.2.1)
   gemnasium-parser (= 0.1.9)
-  octokit (= 1.22.0)
-  xlsx_writer
+  gems (~> 0.7.0)
+  octokit (~> 1.22.0)
+  xlsx_writer (~> 0.4.0)

deps.rb

@@ -1,7 +1,10 @@
+require 'base64'
 require 'bundler'
-require 'octokit'
 require 'gemnasium/parser'
-require 'base64'
+require 'gems'
+require 'octokit'
+require 'open-uri'
+require 'ostruct'
 require 'xlsx_writer'
 
 module Gemnasium
@@ -15,69 +18,206 @@ def exclude?(match, opts)
   end
 end
 
+module Bundler
+  class Dependency
+    attr_accessor :line # written to by Gemnasium::Parser and read by me when doing a git blame!
+    attr_accessor :committer # written to by me to store who last touched the Gemfile at this line.
+  end
+end
+
+class RepoData < Struct.new(:owner, :name, :branch)
+  attr_accessor :dependencies
+
+  @@info_cache = {}
+
+  def dependencies
+    @dependencies ||= []
+  end
+
+  def full_name
+    "#{owner}/#{name}"
+  end
+
+  def specs
+    @specs ||= {}
+  end
+
+  def github_url(spec)
+    if spec.source.to_s =~ /^[email protected]:(.+).git \(at (.+)\)$/ # => https://github.com/$1/tree/$2
+      "http://github.com/#{$1}/tree/#{$2}"
+    else
+      (@@info_cache[spec.name] ||= Gems.info(spec.name))['source_code_uri']
+    end
+  end
+
+  def git_reference(spec)
+    if spec.source.to_s =~ /^[email protected]:(.+).git \(at (.+)\)$/ # => https://github.com/$1/tree/$2
+      $2
+    end
+  end
+
+  def make_a_gem(spec)
+    OpenStruct.new(
+      :name => spec.name,
+      :version => spec.version.to_s,
+      :github_url => github_url(spec),
+      :git_reference => git_reference(spec),
+      :source => spec.source,
+      :spec => spec
+    )
+  end
+
+  def refined_gems
+    puts "Refining gems for #{self}"
+    gems = []
+    # all_gem_dependencies = Marshal.load(open("http://rubygems.org/api/v1/dependencies?gems=#{dependencies.collect(&:name).join(',')}"))
+
+    gem_groups = dependencies.each_with_object({}) do |dependency, hash|
+      spec = specs[dependency.name]
+      if spec && gemmy = make_a_gem(spec)
+        # gem_dependencies_hash = all_gem_dependencies.detect{|h| h[:name] == gemmy.name && h[:version] == gemmy.version}
+        # if gem_dependencies_hash.nil?
+        #   if gemmy.github_url =~ /github.com\/([^\/]+\/[^\/]+)/ # client.contents(github_url(spec)[])
+        #     # We have a github url so we know the repo full name (e.g. github.com/rails/rails => rails/rails)
+        #     begin
+        #       gemspec = Gemnasium::Parser.gemspec(decoded_content(
+        #         client.contents($1, :path => "#{gemmy.name}.gemspec", :ref => gemmy.git_reference).content
+        #       ))
+        #       gemmy.dependent_gems = gemspec.dependencies.collect{|dep| specs[dep.name]}
+        #     rescue
+        #       debugger
+        #       gemmy.dependent_gems = []
+        #     end
+        #   else
+        #     # ¯\_(ツ)_/¯.
+        #   end
+        # else
+        #   gemmy.dependent_gems = gem_dependencies_hash[:dependencies].collect{|(name, requirement)| make_a_gem(specs[name])}
+        # end
+        gemmy.dependent_gems = spec.dependencies.collect do |dep|
+          if specs[dep.name]
+            make_a_gem(specs[dep.name])
+          else
+            puts "Hm, looks like we don't have a spec for #{dep.name} for #{gemmy.name}"
+          end
+        end.compact
+        dependency.groups.each{|group| (hash[group] ||= []).concat([gemmy, *gemmy.dependent_gems])}
+      else
+        puts "Oh noes, no spec for #{dependency.name}"
+      end
+    end
+  end
+
+  def to_s
+    "#{name}@#{branch}"
+  end
+end
+
 
 def client
   @client ||= Octokit::Client.new(:login => ENV['GITHUB_USERNAME'], :oauth_token => ENV['GITHUB_AUTH_TOKEN'])
 end
 
-def gems
+def repos_using_gem
   @gems ||= {}
 end
 
-def version_of(dependency)
-  dependency.send(dependency.respond_to?(:version) ? :version : :requirement).to_s
-end
-
 def add_dependency_for(repo, dependency)
-  (gems[ [dependency.name, version_of(dependency), dependency.source.to_s] ] ||= []) << repo
+  repo.specs[dependency.name] = dependency
+  (repos_using_gem[dependency.name] ||= []) << repo
 end
 
 def decoded_content(content)
   # content.encoding.eql?('base64') ? Base64.decode64(content) : content
   Base64.decode64(content)
 end
 
+def all_repo_data
+  @repo_data ||= []
+end
+
 def note_dependencies_for(repo, branch_name)
   lock_blob = client.contents(repo.full_name, :path => 'Gemfile.lock', :ref => branch_name) rescue nil
   gemfile_blob = client.contents(repo.full_name, :path => 'Gemfile', :ref => branch_name) rescue nil
-  if lock_blob# && gemfile_blob
-    puts "Parsing #{repo.full_name}@#{branch_name}'s Gemfile.lock"
+  repo_data = RepoData.new(repo.owner.login, repo.name, branch_name)
+
+  if gemfile_blob
     # Gem groups can be collected from the Gemfile
-    # gemfile = Gemnasium::Parser.gemfile(decoded_content(gemfile_blob))
-    # gem_groups = gemfile.dependencies.inject({}){|h, g| h.merge!(g.name => g.groups)}
-    # gemfile.dependencies.each{|dep| add_dependency_for("#{repo.name}@#{branch_name}", dep)}
+    puts "Parsing #{repo.full_name}@#{branch_name}'s Gemfile"
+    gemfile = Gemnasium::Parser.gemfile(decoded_content(gemfile_blob.content))
+    repo_data.dependencies = gemfile.dependencies
+  else
+    puts "Couldn't find a Gemfile in #{repo.full_name}@#{branch_name}"
+  end
+
+  if lock_blob
+    puts "Parsing #{repo.full_name}@#{branch_name}'s Gemfile.lock"
     lockfile = Bundler::LockfileParser.new(decoded_content(lock_blob.content))
-    lockfile.specs.each{|spec| add_dependency_for("#{repo.name}@#{branch_name}", spec)}
+    lockfile.specs.each{|spec| add_dependency_for(repo_data, spec)}
   else
     puts "Couldn't find a Gemfile.lock in #{repo.full_name}@#{branch_name}"
   end
+  all_repo_data << repo_data if gemfile_blob && lock_blob
 end
 
-client.organization_repositories('mdsol').each do |repo|
-  ['master', 'develop', 'release'].each do |branch_name|
-    note_dependencies_for(repo, branch_name)
+def gems_by_group
+  @gems_by_group ||= Hash.new{|h, k| h[k] = Set.new}
+end
+
+if ENV['READ_DUMP']
+  @repo_data = Marshal.load(File.read('all_repo_data.dump'))
+else
+  client.organization_repositories('mdsol').each do |repo|
+    ['master', 'develop', 'release'].each do |branch_name|
+      note_dependencies_for(repo, branch_name)
+    end
   end
+  File.open('all_repo_data.dump', 'w'){|f| f.print Marshal.dump(all_repo_data)}
 end
 
-puts "Gem count: #{gems.keys.count}"
+all_repo_data.each do |repo_data|
+  repo_data.refined_gems.each_pair do |group, gems|
+    gems_by_group[group].merge(gems)
+  end
+end
 
-doc = XlsxWriter::Document.new
-sheet1 = doc.add_sheet 'Used in Released Products'
-sheet1.add_row ['Free & Open Source Software (FOSS) Usage [in Medidata Products]']
-sheet1.add_row []
-sheet1.add_row ['FOSS Software Name', 'Version', 'Requestor', 'License', 'For Use in Medidata Products', 'URL of Software Application', 'Internal Repository']
-sheet1.add_autofilter 'A3:G3'
-gems.sort_by{|((gem_name, _), _)| gem_name}.each do |(gem_name, gem_version, gem_source), repos_using_gem|
-  sheet1.add_row [
-    gem_name,
-    gem_version,
-    '', # Requestor
-    '', # License
-    repos_using_gem.join(', '),
-    gem_source,
-    (gem_source && "#{gem_source}".include?('github.com:mdsol/')) ? 'yes' : 'no'
+prod_gems = gems_by_group[:default]
+dev_gems = gems_by_group.each_with_object(Set.new) do |(group, gems), set|
+  set.merge(group == :default ? [] : gems)
+end
+
+puts "Gem count: prod - #{prod_gems.count}, dev - #{dev_gems.count}"
+
+def fill_sheet(sheet, gems)
+  sheet.add_row ['Free & Open Source Software (FOSS) Usage [in Medidata Products]']
+  sheet.add_row []
+  sheet.add_row [
+    'FOSS Software Name',
+    'Version',
+    'Requestor',
+    'License',
+    'For Use in Medidata Products',
+    'URL of Software Application',
+    'Internal Repository'
   ]
+  sheet.add_autofilter 'A3:G3'
+  gems.sort_by(&:name).each do |gemmy|
+    sheet.add_row [
+      gemmy.name,
+      gemmy.version,
+      '', # Requestor
+      '', # License
+      repos_using_gem[gemmy.name].join(', '),
+      "#{gemmy.github_url.blank? ? gemmy.source : gemmy.github_url}",
+      (gemmy.source && "#{gemmy.source}".include?('github.com:mdsol/')) ? 'yes' : 'no'
+    ]
+  end
 end
+
+doc = XlsxWriter::Document.new
+fill_sheet(doc.add_sheet('Used in Released Products'), prod_gems)
+fill_sheet(doc.add_sheet('Used Internally at Medidata'), dev_gems)
+
 require 'fileutils'
 FileUtils.mv doc.path, "./foss.xlsx"
 doc.cleanup