Merge pull request puppetlabs#5691 from thallgren/issues/pup-7305/hie…

…ra-file-cache

(PUP-7305) Add file cache to Hiera

benchmarks/hiera_include_one/benchmarker.rb

@@ -0,0 +1,99 @@
+require 'fileutils'
+
+class Benchmarker
+  include FileUtils
+
+  def initialize(target, size)
+    @target = target
+    @size = size > 1000 ? size : 1000
+  end
+
+  def setup
+    require 'puppet'
+    @config = File.join(@target, 'puppet.conf')
+    Puppet.initialize_settings(['--config', @config])
+    envs = Puppet.lookup(:environments)
+    @node = Puppet::Node.new('testing', :environment => envs.get('benchmarking'))
+  end
+
+  def run(args=nil)
+    @size.times do
+      @compiler = Puppet::Parser::Compiler.new(@node)
+      @compiler.compile do |catalog|
+        scope = @compiler.topscope
+        scope['confdir'] = 'test'
+          hiera_func = @compiler.loaders.puppet_system_loader.load(:function, 'hiera_include')
+          hiera_func.call(scope, 'common_entry')
+        catalog
+      end
+    end
+  end
+
+  def generate
+    env_dir = File.join(@target, 'environments', 'benchmarking')
+    manifests_dir = File.join(env_dir, 'manifests')
+    dummy_class_manifest = File.join(manifests_dir, 'foo.pp')
+    hiera_yaml = File.join(@target, 'hiera.yaml')
+    datadir = File.join(@target, 'data')
+    localdir = File.dirname(File.realpath(__FILE__))
+    common_yaml = File.join(datadir, 'common.yaml')
+    groups_yaml = File.join(datadir, 'groups.yaml')
+
+    mkdir_p(env_dir)
+    mkdir_p(manifests_dir)
+    mkdir_p(datadir)
+
+    File.open(hiera_yaml, 'w') do |f|
+      f.puts(<<-YAML)
+---
+:backends: yaml
+:yaml:
+   :datadir: #{datadir}
+:hierarchy:
+   - common
+   - groups
+:logger: noop
+      YAML
+    end
+
+    File.open(groups_yaml, 'w') do |f|
+      f.puts(<<-YAML)
+---
+puppet:
+  staff:
+    groups:
+      YAML
+
+      0.upto(50).each do |i|
+        f.puts("      group#{i}:")
+        0.upto(125).each do |j|
+          f.puts("        - user#{j}")
+        end
+      end
+    end
+
+    File.open(dummy_class_manifest, 'w') do |f|
+      f.puts("class dummy_class { }")
+    end
+
+    File.open(common_yaml, 'w') do |f|
+      f.puts(<<-YAML)
+        common_entry:
+          - dummy_class
+      YAML
+    end
+
+    templates = File.dirname(File.realpath(__FILE__))
+
+    render(File.join(templates, 'puppet.conf.erb'),
+      File.join(@target, 'puppet.conf'),
+      :location => @target)
+  end
+
+  def render(erb_file, output_file, bindings)
+    site = ERB.new(File.read(erb_file))
+    File.open(output_file, 'w') do |fh|
+      fh.write(site.result(OpenStruct.new(bindings).instance_eval { binding }))
+    end
+  end
+end

benchmarks/hiera_include_one/description

@@ -0,0 +1,2 @@
+Benchmark scenario: Large nested hierarchy dataset, many compilations, one lookup per compile
+Benchmark target: hiera include.

benchmarks/hiera_include_one/puppet.conf.erb

@@ -0,0 +1,5 @@
+confdir = <%= location %>
+vardir = <%= location %>
+codedir = <%= location %>
+environmentpath = <%= File.join(location, 'environments') %>
+environment_timeout = 0

lib/puppet/functions/eyaml_lookup_key.rb

@@ -28,21 +28,29 @@ def eyaml_lookup_key(key, options, context)
           context.explain { "Setting Eyaml option '#{k}' to '#{v}'" }
         end
       end
-      raw_data = load_data_hash(options)
+      raw_data = load_data_hash(options, context)
       context.cache(nil, raw_data)
     end
     context.not_found unless raw_data.include?(key)
     context.cache(key, decrypt_value(raw_data[key], context))
   end
 
-  def load_data_hash(options)
-    begin
-      data = YAML.load_file(options['path'])
-      Puppet::Pops::Lookup::HieraConfig.symkeys_to_string(data.is_a?(Hash) ? data : {})
-    rescue YAML::SyntaxError => ex
-      # Psych errors includes the absolute path to the file, so no need to add that
-      # to the message
-      raise Puppet::DataBinding::LookupError, "Unable to parse #{ex.message}"
+  def load_data_hash(options, context)
+    path = options['path']
+    context.cached_file_data(path) do |content|
+      begin
+        data = YAML.load(content, path)
+        if data.is_a?(Hash)
+          Puppet::Pops::Lookup::HieraConfig.symkeys_to_string(data)
+        else
+          Puppet.warning("#{path}: file does not contain a valid yaml hash")
+          {}
+        end
+      rescue YAML::SyntaxError => ex
+        # Psych errors includes the absolute path to the file, so no need to add that
+        # to the message
+        raise Puppet::DataBinding::LookupError, "Unable to parse #{ex.message}"
+      end
     end
   end
 

lib/puppet/functions/hocon_data.rb

@@ -15,10 +15,12 @@
 
   def hocon_data(options, context)
     path = options['path']
-    begin
-      Hocon.parse(Puppet::FileSystem.read(path, :encoding => 'utf-8'))
-    rescue Hocon::ConfigError => ex
-      raise Puppet::DataBinding::LookupError, "Unable to parse (#{path}): #{ex.message}"
+    context.cached_file_data(path) do |content|
+      begin
+        Hocon.parse(content)
+      rescue Hocon::ConfigError => ex
+        raise Puppet::DataBinding::LookupError, "Unable to parse (#{path}): #{ex.message}"
+      end
     end
   end
 end

lib/puppet/functions/json_data.rb

@@ -8,11 +8,13 @@
 
   def json_data(options, context)
     path = options['path']
-    begin
-      JSON.parse(Puppet::FileSystem.read(path, :encoding => 'utf-8'))
-    rescue JSON::ParserError => ex
-      # Filename not included in message, so we add it here.
-      raise Puppet::DataBinding::LookupError, "Unable to parse (#{path}): #{ex.message}"
+    context.cached_file_data(path) do |content|
+      begin
+        JSON.parse(content)
+      rescue JSON::ParserError => ex
+        # Filename not included in message, so we add it here.
+        raise Puppet::DataBinding::LookupError, "Unable to parse (#{path}): #{ex.message}"
+      end
     end
   end
 end

lib/puppet/functions/yaml_data.rb

@@ -9,18 +9,21 @@
   end
 
   def yaml_data(options, context)
-    begin
-      path = options['path']
-      data = YAML.load_file(path)
-      unless data.is_a?(Hash)
-        Puppet.warning("#{path}: file does not contain a valid yaml hash")
-        data = {}
+    path = options['path']
+    context.cached_file_data(path) do |content|
+      begin
+        data = YAML.load(content, path)
+        if data.is_a?(Hash)
+          Puppet::Pops::Lookup::HieraConfig.symkeys_to_string(data)
+        else
+          Puppet.warning("#{path}: file does not contain a valid yaml hash")
+          {}
+        end
+      rescue YAML::SyntaxError => ex
+        # Psych errors includes the absolute path to the file, so no need to add that
+        # to the message
+        raise Puppet::DataBinding::LookupError, "Unable to parse #{ex.message}"
       end
-      Puppet::Pops::Lookup::HieraConfig.symkeys_to_string(data.nil? ? {} : data)
-    rescue YAML::SyntaxError => ex
-      # Psych errors includes the absolute path to the file, so no need to add that
-      # to the message
-      raise Puppet::DataBinding::LookupError, "Unable to parse #{ex.message}"
     end
   end
 end

lib/puppet/pops/lookup/configured_data_provider.rb

@@ -13,7 +13,7 @@ def initialize(config = nil)
   end
 
   def config(lookup_invocation)
-    @config ||= assert_config_version(HieraConfig.create(configuration_path(lookup_invocation)))
+    @config ||= assert_config_version(HieraConfig.create(lookup_invocation, configuration_path(lookup_invocation)))
   end
 
   # @return [Pathname] the path to the configuration

lib/puppet/pops/lookup/context.rb

@@ -2,19 +2,73 @@
 
 module Puppet::Pops
 module Lookup
+# The EnvironmentContext is adapted to the current environment
+#
+class EnvironmentContext < Adaptable::Adapter
+  class FileData
+    attr_reader :data
+
+    def initialize(path, inode, mtime, size, data)
+      @path = path
+      @inode = inode
+      @mtime = mtime
+      @size = size
+      @data = data
+    end
+
+    def valid?(stat)
+      stat.ino == @inode && stat.mtime == @mtime && stat.size == @size
+    end
+  end
+
+  attr_reader :environment_name
+
+  def self.create_adapter(environment)
+    new(environment)
+  end
+
+  def initialize(environment)
+    @environment_name = environment.name
+    @file_data_cache = {}
+  end
+
+  # Loads the contents of the file given by _path_. The content is then yielded to the provided block in
+  # case a block is given, and the returned value from that block is cached and returned by this method.
+  # If no block is given, the content is stored instead.
+  #
+  # The cache is retained as long as the inode, mtime, and size of the file remains unchanged.
+  #
+  # @param path [String] path to the file to be read
+  # @yieldparam content [String] the content that was read from the file
+  # @yieldreturn [Object] some result based on the content
+  # @return [Object] the content, or if a block was given, the return value of the block
+  #
+  def cached_file_data(path)
+    file_data = @file_data_cache[path]
+    stat = Puppet::FileSystem.stat(path)
+    unless file_data && file_data.valid?(stat)
+      Puppet.debug("File at '#{path}' was changed, reloading") if file_data
+      content = Puppet::FileSystem.read(path, :encoding => 'utf-8')
+      file_data = FileData.new(path, stat.ino, stat.mtime, stat.size, block_given? ? yield(content) : content)
+      @file_data_cache[path] = file_data
+    end
+    file_data.data
+  end
+end
+
 # A FunctionContext is created for each unique hierarchy entry and adapted to the Compiler (and hence shares
 # the compiler's life-cycle).
 # @api private
 class FunctionContext
   include Interpolation
 
-  attr_reader :environment_name, :module_name, :function
+  attr_reader :module_name, :function
   attr_accessor :data_hash
 
-  def initialize(environment_name, module_name, function)
+  def initialize(environment_context, module_name, function)
     @data_hash = nil
     @cache = {}
-    @environment_name = environment_name
+    @environment_context = environment_context
     @module_name = module_name
     @function = function
   end
@@ -51,6 +105,14 @@ def cached_entries(&block)
       Types::Iterable.on(@cache)
     end
   end
+
+  def cached_file_data(path, &block)
+    @environment_context.cached_file_data(path, &block)
+  end
+
+  def environment_name
+    @environment_context.environment_name
+  end
 end
 
 # The Context is created once for each call to a function. It provides a combination of the {Invocation} object needed
@@ -73,10 +135,12 @@ def self.register_ptype(loader, ir)
     key_type = tf.optional(tf.scalar)
     @type = Pcore::create_object_type(loader, ir, self, 'Puppet::LookupContext', 'Any',
       {
-        'environment_name' => Types::PStringType::NON_EMPTY,
+        'environment_name' => {
+          Types::KEY_TYPE => Types::PStringType::NON_EMPTY,
+          Types::KEY_KIND => Types::PObjectType::ATTRIBUTE_KIND_DERIVED
+        },
         'module_name' => {
-          Types::KEY_TYPE => tf.optional(Types::PStringType::NON_EMPTY),
-          Types::KEY_VALUE => nil
+          Types::KEY_TYPE => tf.variant(Types::PStringType::NON_EMPTY, Types::PUndefType::DEFAULT)
         }
       },
       {
@@ -90,19 +154,19 @@ def self.register_ptype(loader, ir)
         'cached_entries' => tf.variant(
           tf.callable([0, 0, tf.callable(1,1)], tf.undef),
           tf.callable([0, 0, tf.callable(2,2)], tf.undef),
-          tf.callable([0, 0], tf.iterable(tf.tuple([key_type, tf.any])))
-        )
+          tf.callable([0, 0], tf.iterable(tf.tuple([key_type, tf.any])))),
+        'cached_file_data' => tf.callable(tf.string, tf.optional(tf.callable([1, 1])))
       }
     ).resolve(Types::TypeParser.singleton, loader)
   end
 
   # Mainly for test purposes. Makes it possible to create a {Context} in Puppet code provided that a current {Invocation} exists.
-  def self.from_asserted_args(environment_name, module_name)
-    new(FunctionContext.new(environment_name, module_name, nil), Invocation.current)
+  def self.from_asserted_args(module_name)
+    new(FunctionContext.new(EnvironmentContext.adapt(Puppet.lookup(:environments).get(Puppet[:environment])), module_name, nil), Invocation.current)
   end
 
   # Public methods delegated to the {FunctionContext}
-  def_delegators :@function_context, :cache, :cache_all, :cache_has_key, :cached_value, :cached_entries, :environment_name, :module_name
+  def_delegators :@function_context, :cache, :cache_all, :cache_has_key, :cached_value, :cached_entries, :environment_name, :module_name, :cached_file_data
 
   def initialize(function_context, lookup_invocation)
     @lookup_invocation = lookup_invocation

lib/puppet/pops/lookup/function_provider.rb

@@ -21,9 +21,12 @@ def initialize(name, parent_data_provider, function_name, options, locations)
 
   # @return [FunctionContext] the function context associated with this provider
   def function_context(lookup_invocation, location)
+    @contexts[location] ||= create_function_context(lookup_invocation)
+  end
+
+  def create_function_context(lookup_invocation)
     scope = lookup_invocation.scope
-    compiler = scope.compiler
-    @contexts[location] ||= FunctionContext.new(compiler.environment.name, module_name, function(scope))
+    FunctionContext.new(EnvironmentContext.adapt(scope.compiler.environment), module_name, function(scope))
   end
 
   def module_name