Merge branch 'CHEF-799' of git://github.com/danielsdeleo/chef into danielsdeleo/CHEF-799

adamhjk · adamhjk · commit 053cd7295420 · 2010-01-04T14:35:09.000-08:00
diff --git a/Rakefile b/Rakefile
@@ -419,6 +419,10 @@ namespace :features do
       t.profile = "provider_template"
     end
     
+    Cucumber::Rake::Task.new(:remote_directory) do |t|
+      t.profile = "provider_remote_directory"
+    end
+    
     Cucumber::Rake::Task.new(:git) do |t|
       t.profile = "provider_git"
     end
diff --git a/chef-server-api/app/controllers/cookbooks.rb b/chef-server-api/app/controllers/cookbooks.rb
@@ -65,7 +65,11 @@ def show_segment
     if params[:id]
       case params[:segment]
       when "templates","files"
-        serve_segment_preferred(cookbook, params[:segment], cookbook_files[params[:segment].to_sym])
+        if params[:recursive]
+          serve_directory_preferred(cookbook, params[:segment], cookbook_files[params[:segment].to_sym])
+        else
+          serve_segment_preferred(cookbook, params[:segment], cookbook_files[params[:segment].to_sym])
+        end
       else
         serve_segment_file(cookbook, params[:segment], cookbook_files[params[:segment].to_sym])
       end
@@ -78,20 +82,19 @@ def serve_segment_preferred(cookbook, segment, files)
 
     to_send = nil
 
-    preferences = [
-      "host-#{params[:fqdn]}",
-      "#{params[:platform]}-#{params[:version]}",
-      "#{params[:platform]}",
-      "default"
-    ]
-
+    
     preferences.each do |pref|
       unless to_send
-        to_send = files.detect { |file| Chef::Log.debug("#{pref.inspect} #{file.inspect}"); file[:name] == params[:id] && file[:specificity] == pref }
+        Chef::Log.debug("Looking for a file with name `#{params[:id]}' and specificity #{pref}")
+        to_send = files.detect do |file| 
+          Chef::Log.debug("#{pref.inspect} #{file.inspect}")
+          file[:name] == params[:id] && file[:specificity] == pref
+          
+        end
       end
     end
 
-    raise NotFound, "Cannot find a suitable #{segment} file!" unless to_send 
+    raise NotFound, "Cannot find a suitable #{segment} file for #{params[:id]}!" unless to_send 
     current_checksum = to_send[:checksum] 
     Chef::Log.debug("#{to_send[:name]} Client Checksum: #{params[:checksum]}, Server Checksum: #{current_checksum}") 
     if current_checksum == params[:checksum]
@@ -108,6 +111,25 @@ def serve_segment_preferred(cookbook, segment, files)
       send_file(file_name)
     end
   end
+  
+  def serve_directory_preferred(cookbook, segment, files)
+    preferred_dir_contents = []
+    preferences.each do |preference|
+      preferred_dir_contents = files.select { |file| file[:name] =~ /^#{params[:id]}/ && file[:specificity] == preference  }
+      break unless preferred_dir_contents.empty?
+    end
+    
+    raise NotFound, "Cannot find a suitable directory for #{params[:id]}" if preferred_dir_contents.empty?
+    
+    display preferred_dir_contents.map { |file| file[:name].sub(/^#{params[:id]}/, '') }
+  end
+  
+  def preferences
+    ["host-#{params[:fqdn]}",
+    "#{params[:platform]}-#{params[:version]}",
+    "#{params[:platform]}",
+    "default"]
+  end
 
   def serve_segment_file(cookbook, segment, files)
     to_send = files.detect { |f| f[:name] == params[:id] } 
diff --git a/chef/lib/chef/provider/remote_directory.rb b/chef/lib/chef/provider/remote_directory.rb
@@ -35,8 +35,6 @@ class RemoteDirectory < Chef::Provider::Directory
       
       def action_create
         super
-
-        @remote_file_list = Hash.new
         do_recursive
       end
     
@@ -45,81 +43,92 @@ def action_create
       def do_recursive
         if Chef::Config[:solo]
           Chef::Log.debug("Doing a local recursive directory copy for #{@new_resource}")
-          files_to_transfer = files_for_directory(@new_resource.source)
         else
           Chef::Log.debug("Doing a remote recursive directory transfer for #{@new_resource}")
-          r = Chef::REST.new(Chef::Config[:remotefile_url])
-          files_to_transfer = r.get_rest(generate_url(@new_resource.source, "files", { :recursive => "true" }))
-        end        
-
+        end
+          
         files_to_transfer.each do |remote_file_source|
           fetch_remote_file(remote_file_source)
         end
       end
       
+      def files_to_transfer
+        file_list = Chef::Config[:solo] ? generate_solo_file_list : generate_client_file_list
+        Chef::Log.debug("Generated file manifest for #{@new_resource}:\n#{file_list.join("\n")}")
+        file_list
+      end
+      
+      def generate_solo_file_list
+        # Pulled from chef-server-slice files controller
+        directory = find_preferred_file(
+          @new_resource.cookbook_name, 
+          :remote_file, 
+          @new_resource.source, 
+          @node[:fqdn],
+          @node[:platform],
+          @node[:platform_version]
+        )
+
+        unless (directory && ::File.directory?(directory))
+          raise NotFound, "Cannot find a suitable directory"
+        end
+
+        Dir[::File.join(directory, '**', '*')].sort.reverse.select do |file|
+          file[/^#{directory}\/(.+)$/, 1] unless ::File.directory?(file)
+        end
+      end
+      
+      def generate_client_file_list
+        r = Chef::REST.new(Chef::Config[:remotefile_url])
+        r.get_rest(generate_url(@new_resource.source, "files", { :recursive => "true" }))
+      end
+      
       def fetch_remote_file(remote_file_source)
         full_path = ::File.join(@new_resource.path, remote_file_source)
-        full_dir = ::File.dirname(full_path)
         
-        if !::File.directory?(full_dir)
-          create_directory(full_dir)
-        end
-
-        remote_file = Chef::Resource::RemoteFile.new(full_path, nil, @node)
-        remote_file.cookbook_name = @new_resource.cookbook || @new_resource.cookbook_name           
-        remote_file.source(::File.join(@new_resource.source, remote_file_source))
+        ensure_directory_exists(::File.dirname(full_path))
+        
+        file_to_fetch = provider_for_remote_file(full_path, remote_file_source)
+        file_to_fetch.load_current_resource
+        file_to_fetch.action_create
+        @new_resource.updated = true if file_to_fetch.new_resource.updated        
+      end
+      
+      def provider_for_remote_file(path, source)
+        remote_file = Chef::Resource::RemoteFile.new(path, nil, @node)
+        remote_file.cookbook_name = @new_resource.cookbook || @new_resource.cookbook_name
+        remote_file.source(::File.join(@new_resource.source, source))
         remote_file.mode(@new_resource.files_mode) if @new_resource.files_mode
         remote_file.group(@new_resource.files_group) if @new_resource.files_group
         remote_file.owner(@new_resource.files_owner) if @new_resource.files_owner
         remote_file.backup(@new_resource.files_backup) if @new_resource.files_backup
         
-        rf_provider = Chef::Platform.provider_for_node(@node, remote_file)
-        rf_provider.load_current_resource
-        rf_provider.action_create
-        @new_resource.updated = true if rf_provider.new_resource.updated        
+        Chef::Platform.provider_for_node(@node, remote_file)
       end
       
-      def create_directory(full_dir)
-        new_dir = Chef::Resource::Directory.new(full_dir, nil, @node)
+      def ensure_directory_exists(path)
+        unless ::File.directory?(path)
+          directory_to_create = provider_for_directory(path)
+          directory_to_create.load_current_resource
+          directory_to_create.action_create
+          @new_resource.updated = true if directory_to_create.new_resource.updated  
+        end
+      end
+      
+      def provider_for_directory(path)
+        new_dir = Chef::Resource::Directory.new(path, nil, @node)
         new_dir.cookbook_name = @new_resource.cookbook || @new_resource.cookbook_name            
         new_dir.mode(@new_resource.mode)
         new_dir.group(@new_resource.group)
         new_dir.owner(@new_resource.owner)
         new_dir.recursive(true)
         
-        d_provider = Chef::Platform.provider_for_node(@node, new_dir)
-        d_provider.load_current_resource
-        d_provider.action_create
-        @new_resource.updated = true if d_provider.new_resource.updated  
+        Chef::Platform.provider_for_node(@node, new_dir)
       end
             
       def action_create_if_missing
         raise Chef::Exceptions::UnsupportedAction, "Remote Directories do not support create_if_missing."
       end
-      # Pulled from chef-server-slice files controller
-
-      def files_for_directory(path)
-        directory = find_preferred_file(
-          @new_resource.cookbook_name, 
-          :remote_file, 
-          path, 
-          @node[:fqdn],
-          @node[:platform],
-          @node[:platform_version]
-        )
-
-        unless (directory && ::File.directory?(directory))
-          raise NotFound, "Cannot find a suitable directory"
-        end
-
-        directory_listing = Array.new
-        Dir[::File.join(directory, '**', '*')].sort { |a,b| b <=> a }.each do |file|
-          next if ::File.directory?(file)
-          file =~ /^#{directory}\/(.+)$/
-          directory_listing << $1
-        end
-        directory_listing
-      end
 
     end
   end
diff --git a/chef/spec/data/remote_directory_data/remote_dir_file.txt b/chef/spec/data/remote_directory_data/remote_dir_file.txt
@@ -0,0 +1 @@
+I'm a file inside a the root remote_directory source dir.
diff --git a/chef/spec/data/remote_directory_data/remote_subdirectory/remote_subdir_file.txt b/chef/spec/data/remote_directory_data/remote_subdirectory/remote_subdir_file.txt
@@ -0,0 +1 @@
+I'm a file in a subdirectory inside a remote_directory source
diff --git a/chef/spec/unit/platform_spec.rb b/chef/spec/unit/platform_spec.rb
@@ -19,6 +19,14 @@
 require File.expand_path(File.join(File.dirname(__FILE__), "..", "spec_helper"))
 
 describe Chef::Platform do
+  before :all do
+    @original_platform_map = Chef::Platform.platforms
+  end
+  
+  after :all do ||
+    Chef::Platform.platforms = @original_platform_map
+  end
+  
   before(:each) do
     Chef::Platform.platforms = {
       :darwin => {
diff --git a/chef/spec/unit/provider/remote_directory_spec.rb b/chef/spec/unit/provider/remote_directory_spec.rb
@@ -0,0 +1,132 @@
+#
+# Author:: Daniel DeLeo (<dan@kallistec.com>)
+# Copyright:: Copyright (c) 2010 Daniel DeLeo
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "spec_helper"))
+
+describe Chef::Provider::RemoteDirectory do
+  before do
+    @resource = Chef::Resource::RemoteDirectory.new("/tmp/tafty")
+    @resource.source "path/on/server"
+    @node = Chef::Node.new
+    @node.name "latte"
+    @node.platform :mac_os_x
+    @node.platform_version "10.6"
+    @provider = Chef::Provider::RemoteDirectory.new(@node, @resource)
+    @provider.current_resource = @resource.clone
+  end
+  
+  it "doesn't support create_if_missing and explodes if you try to use it" do
+    lambda {@provider.send :action_create_if_missing}.should raise_error(Chef::Exceptions::UnsupportedAction)
+  end
+  
+  describe "recursively transferring files" do
+    before do
+      @resource.mode  "0750"
+      @resource.group "wheel"
+      @resource.owner "root"
+      
+      @resource.files_mode  "0640"
+      @resource.files_group "staff"
+      @resource.files_owner "toor"
+      @resource.files_backup 23
+      
+      @provider.current_resource = @resource.clone
+    end
+    
+    it "creates Directory resources and assigns them the correct attributes" do
+      directory_resource = @provider.send(:provider_for_directory, "/tmp/intermediate_dir").new_resource
+      directory_resource.path.should  == "/tmp/intermediate_dir"
+      directory_resource.mode.should  == "0750"
+      directory_resource.group.should == "wheel"
+      directory_resource.owner.should == "root"
+      directory_resource.recursive.should be_true
+    end
+    
+    it "creates intermediate directories as required" do
+      @directory_resource = mock("Resource::Directory", :updated => true)
+      @directory_provider = mock("Provider::Directory", :new_resource => @directory_resource)
+      @directory_provider.should_receive(:load_current_resource)
+      @directory_provider.should_receive(:action_create)
+      @provider.stub!(:provider_for_directory).and_return(@directory_provider)
+      @provider.send(:ensure_directory_exists, "/tmp/an_intermediate_dir")
+      @resource.updated.should be_true
+    end
+    
+    it "creates remote_file resources and assigns them the correct attributes" do
+      @resource.cookbook "berlin_style_tasty_cupcakes"
+      rf_provider = @provider.send(:provider_for_remote_file, 
+                                    "/enclosing_dir/file_to_transfer.txt", 
+                                    "file_to_transfer.txt")
+      rf_resource = rf_provider.new_resource
+      rf_resource.cookbook_name.should  == "berlin_style_tasty_cupcakes"
+      rf_resource.source.should         == "path/on/server/file_to_transfer.txt"
+      rf_resource.mode.should           == "0640"
+      rf_resource.group.should          == "staff"
+      rf_resource.owner.should          == "toor"
+      rf_resource.backup.should         == 23
+    end
+    
+    it "fetches files using remote file resources" do
+      @rf_resource = mock("Resource::RemoteFile", :updated => true)
+      @rf_provider = mock("Provider::RemoteFile", :new_resource => @rf_resource)
+      
+      @rf_provider.should_receive(:load_current_resource)
+      @rf_provider.should_receive(:action_create)
+      
+      @provider.stub!(:ensure_directory_exists)
+      @provider.stub!(:provider_for_remote_file).and_return(@rf_provider)
+      
+      @provider.send(:fetch_remote_file, "foo")
+      @resource.updated.should be_true
+    end
+  end
+  
+  describe "generating the list of files to transfer" do
+    
+    after do
+      Chef::Config[:solo] = false
+    end
+    
+    it "lists the directory contents from the cookbook for chef-solo" do
+      Chef::Config[:solo] = true
+      @source_path = File.join(File.dirname(__FILE__), "..", "..", "data", "remote_directory_data")
+      @resource.source(@source_path)
+      @provider.stub!(:find_preferred_file).and_return(@source_path)
+      
+      file_list = @provider.send(:generate_solo_file_list)
+      file_list.map! { |f| ::File.expand_path(f) }
+      expected_file_list = %w{ remote_subdirectory/remote_subdir_file.txt remote_dir_file.txt }
+      expected_file_list.map! { |f| ::File.expand_path(::File.join(@source_path, f)) }
+      file_list.should == expected_file_list
+    end
+    
+    it "requests a recursive file listing from the server for chef-client" do
+      @resource.source("dir_on_the_server")
+      Chef::Config[:solo] = false
+      
+      @rest = mock("Chef::REST")
+      Chef::REST.should_receive(:new).and_return(@rest)
+      
+      list = %w{ foo bar baz}
+      @rest.should_receive(:get_rest).with("cookbooks//files?id=dir_on_the_server&recursive=true").and_return(list)
+      @provider.send(:generate_client_file_list).should == list
+      
+    end
+    
+  end
+end
diff --git a/cucumber.yml b/cucumber.yml
@@ -45,7 +45,7 @@ client_run_interval: --tags client_run_interval --format pretty -r features/step
 recipe_inclusion: --tags recipe_inclusion --format pretty -r features/steps -r features/support features 
 attribute_inclusion: --tags @attribute_inclusion --format pretty -r features/steps -r features/support features 
 cookbooks: --tags @cookbooks --format pretty -r features/steps -r features/support features 
-provider_remote_file: --tags provider,remote_file --format pretty -r features/steps -r features/support features 
+provider_remote_directory: --tags @remote_directory --format pretty -r features/steps -r features/support features 
 provider_git: --tags provider,git --format pretty -r features/steps -r features/support features 
 provider_template: --tags template --format pretty -r features/steps -r features/support features
 provider_package_macports: --tags macports --format pretty -r features/steps -r features/support features 
diff --git a/features/data/cookbooks/transfer_remote_directories/files/default/transfer_directory_feature/bar.txt b/features/data/cookbooks/transfer_remote_directories/files/default/transfer_directory_feature/bar.txt
@@ -0,0 +1 @@
+Space Manoeuvres stage 1
diff --git a/features/data/cookbooks/transfer_remote_directories/files/default/transfer_directory_feature/baz.txt b/features/data/cookbooks/transfer_remote_directories/files/default/transfer_directory_feature/baz.txt
@@ -0,0 +1 @@
+micromega
diff --git a/features/data/cookbooks/transfer_remote_directories/files/default/transfer_directory_feature/foo.txt b/features/data/cookbooks/transfer_remote_directories/files/default/transfer_directory_feature/foo.txt
diff --git a/features/data/cookbooks/transfer_remote_directories/recipes/transfer_directory.rb b/features/data/cookbooks/transfer_remote_directories/recipes/transfer_directory.rb
diff --git a/features/provider/remote_directory/transfer_remote_directories.feature b/features/provider/remote_directory/transfer_remote_directories.feature
diff --git a/features/steps/run_client_steps.rb b/features/steps/run_client_steps.rb

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+I'm a file inside a the root remote_directory source dir.`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+I'm a file in a subdirectory inside a remote_directory source`