Merge pull request basecamp#337 from igor-alexandrov/feature/cache

Support for Docker multistage build cache

README.md

@@ -473,6 +473,37 @@ builder:
   context: ".."
 ```
 
+### Using multistage builder cache
+
+Docker multistage build cache can singlehandedly speed up your builds by a lot. Currently MRSK only supports using the GHA cache or the Registry cache:
+
+```yaml
+# Using GHA cache
+builder:
+  cache:
+    type: gha
+
+# Using Registry cache
+builder:
+  cache:
+    type: registry
+
+# Using Registry cache with different cache image
+builder:
+  cache:
+    type: registry
+    # default image name is <image>-build-cache
+    image: application-cache-image
+
+# Using Registry cache with additinonal cache-to options
+builder:
+  cache:
+    type: registry
+    options: mode=max,image-manifest=true,oci-mediatypes=true
+```
+
+For further insights into build cache optimization, check out documentation on Docker's official website: https://docs.docker.com/build/cache/.
+
 ### Using build secrets for new images
 
 Some images need a secret passed in during build time, like a GITHUB_TOKEN, to give access to private gem repositories. This can be done by having the secret in ENV, then referencing it in the builder configuration:

lib/mrsk/commands/builder.rb

@@ -7,11 +7,13 @@ def name
 
   def target
     case
-    when config.builder && config.builder["multiarch"] == false
+    when !config.builder.multiarch? && !config.builder.cached?
       native
-    when config.builder && config.builder["local"] && config.builder["remote"]
+    when !config.builder.multiarch? && config.builder.cached?
+      native_cached
+    when config.builder.local? && config.builder.remote?
       multiarch_remote
-    when config.builder && config.builder["remote"]
+    when config.builder.remote?
       native_remote
     else
       multiarch
@@ -22,6 +24,10 @@ def native
     @native ||= Mrsk::Commands::Builder::Native.new(config)
   end
 
+  def native_cached
+    @native ||= Mrsk::Commands::Builder::Native::Cached.new(config)
+  end
+
   def native_remote
     @native ||= Mrsk::Commands::Builder::Native::Remote.new(config)
   end

lib/mrsk/commands/builder/base.rb

@@ -3,6 +3,7 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
   class BuilderError < StandardError; end
 
   delegate :argumentize, to: Mrsk::Utils
+  delegate :args, :secrets, :dockerfile, :local_arch, :local_host, :remote_arch, :remote_host, :cache_from, :cache_to, to: :builder_config
 
   def clean
     docker :image, :rm, "--force", config.absolute_image
@@ -13,11 +14,11 @@ def pull
   end
 
   def build_options
-    [ *build_tags, *build_labels, *build_args, *build_secrets, *build_dockerfile ]
+    [ *build_tags, *build_cache, *build_labels, *build_args, *build_secrets, *build_dockerfile ]
   end
 
   def build_context
-    context
+    config.builder.context
   end
 
 
@@ -26,6 +27,13 @@ def build_tags
       [ "-t", config.absolute_image, "-t", config.latest_image ]
     end
 
+    def build_cache
+      if cache_to && cache_from
+        ["--cache-to", cache_to,
+          "--cache-from", cache_from]
+      end
+    end
+
     def build_labels
       argumentize "--label", { service: config.service }
     end
@@ -46,19 +54,7 @@ def build_dockerfile
       end
     end
 
-    def args
-      (config.builder && config.builder["args"]) || {}
-    end
-
-    def secrets
-      (config.builder && config.builder["secrets"]) || []
-    end
-
-    def dockerfile
-      (config.builder && config.builder["dockerfile"]) || "Dockerfile"
-    end
-
-    def context
-      (config.builder && config.builder["context"]) || "."
+    def builder_config
+      config.builder
     end
 end

lib/mrsk/commands/builder/multiarch/remote.rb

@@ -22,17 +22,17 @@ def builder_name_with_arch(arch)
     end
 
     def create_local_buildx
-      docker :buildx, :create, "--name", builder_name, builder_name_with_arch(local["arch"]), "--platform", "linux/#{local["arch"]}"
+      docker :buildx, :create, "--name", builder_name, builder_name_with_arch(local_arch), "--platform", "linux/#{local_arch}"
     end
 
     def append_remote_buildx
-      docker :buildx, :create, "--append", "--name", builder_name, builder_name_with_arch(remote["arch"]), "--platform", "linux/#{remote["arch"]}"
+      docker :buildx, :create, "--append", "--name", builder_name, builder_name_with_arch(remote_arch), "--platform", "linux/#{remote_arch}"
     end
 
     def create_contexts
       combine \
-        create_context(local["arch"], local["host"]),
-        create_context(remote["arch"], remote["host"])
+        create_context(local_arch, local_host),
+        create_context(remote_arch, remote_host)
     end
 
     def create_context(arch, host)
@@ -41,19 +41,11 @@ def create_context(arch, host)
 
     def remove_contexts
       combine \
-        remove_context(local["arch"]),
-        remove_context(remote["arch"])
+        remove_context(local_arch),
+        remove_context(remote_arch)
     end
 
     def remove_context(arch)
       docker :context, :rm, builder_name_with_arch(arch)
     end
-
-    def local
-      config.builder["local"]
-    end
-
-    def remote
-      config.builder["remote"]
-    end
 end

lib/mrsk/commands/builder/native.rb

@@ -1,10 +1,10 @@
 class Mrsk::Commands::Builder::Native < Mrsk::Commands::Builder::Base
   def create
-    # No-op on native
+    # No-op on native without cache
   end
 
   def remove
-    # No-op on native
+    # No-op on native without cache
   end
 
   def push

lib/mrsk/commands/builder/native/cached.rb

@@ -0,0 +1,16 @@
+class Mrsk::Commands::Builder::Native::Cached < Mrsk::Commands::Builder::Native
+  def create
+    docker :buildx, :create, "--use", "--driver=docker-container"
+  end
+
+  def remove
+    docker :buildx, :rm, builder_name
+  end
+
+  def push
+    docker :buildx, :build,
+      "--push",
+      *build_options,
+      build_context
+  end
+end

lib/mrsk/commands/builder/native/remote.rb

@@ -28,29 +28,21 @@ def info
 
 
   private
-    def arch
-      config.builder["remote"]["arch"]
-    end
-
-    def host
-      config.builder["remote"]["host"]
-    end
-
     def builder_name
       "mrsk-#{config.service}-native-remote"
     end
 
     def builder_name_with_arch
-      "#{builder_name}-#{arch}"
+      "#{builder_name}-#{remote_arch}"
     end
 
     def platform
-      "linux/#{arch}"
+      "linux/#{remote_arch}"
     end
 
     def create_context
       docker :context, :create,
-        builder_name_with_arch, "--description", "'#{builder_name} #{arch} native host'", "--docker", "'host=#{host}'"
+        builder_name_with_arch, "--description", "'#{builder_name} #{remote_arch} native host'", "--docker", "'host=#{remote_host}'"
     end
 
     def remove_context

lib/mrsk/configuration.rb

@@ -6,7 +6,7 @@
 require "net/ssh/proxy/jump"
 
 class Mrsk::Configuration
-  delegate :service, :image, :servers, :env, :labels, :registry, :builder, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
+  delegate :service, :image, :servers, :env, :labels, :registry, :stop_wait_time, :hooks_path, to: :raw_config, allow_nil: true
   delegate :argumentize, :argumentize_env_with_secrets, :optionize, to: Mrsk::Utils
 
   attr_accessor :destination
@@ -186,7 +186,7 @@ def to_h
       env_args: env_args,
       volume_args: volume_args,
       ssh_options: ssh_options,
-      builder: raw_config.builder,
+      builder: builder.to_h,
       accessories: raw_config.accessories,
       logging: logging_args,
       healthcheck: healthcheck
@@ -201,6 +201,10 @@ def hooks_path
     raw_config.hooks_path || ".mrsk/hooks"
   end
 
+  def builder
+    Mrsk::Configuration::Builder.new(config: self)
+  end
+
   private
     # Will raise ArgumentError if any required config keys are missing
     def ensure_required_keys_present

lib/mrsk/configuration/builder.rb

@@ -0,0 +1,114 @@
+class Mrsk::Configuration::Builder
+  def initialize(config:)
+    @options = config.raw_config.builder || {}
+    @image = config.image
+    @server = config.registry["server"]
+
+    valid?
+  end
+
+  def to_h
+    @options
+  end
+
+  def multiarch?
+    @options["multiarch"] != false
+  end
+
+  def local?
+    !!@options["local"]
+  end
+
+  def remote?
+    !!@options["remote"]
+  end
+
+  def cached?
+    !!@options["cache"]
+  end
+
+  def args
+    @options["args"] || {}
+  end
+
+  def secrets
+    @options["secrets"] || []
+  end
+
+  def dockerfile
+    @options["dockerfile"] || "Dockerfile"
+  end
+
+  def context
+    @options["context"] || "."
+  end
+
+  def local_arch
+    @options["local"]["arch"] if local?
+  end
+
+  def local_host
+    @options["local"]["host"] if local?
+  end
+
+  def remote_arch
+    @options["remote"]["arch"] if remote?
+  end
+
+  def remote_host
+    @options["remote"]["host"] if remote?
+  end
+
+  def cache_from
+    if cached?
+      case @options["cache"]["type"]
+      when "gha"
+        cache_from_config_for_gha
+      when "registry"
+        cache_from_config_for_registry
+      end
+    end
+  end
+
+  def cache_to
+    if cached?
+      case @options["cache"]["type"]
+      when "gha"
+        cache_to_config_for_gha
+      when "registry"
+        cache_to_config_for_registry
+      end
+    end
+  end
+
+  private
+    def valid?
+      if @options["local"] && !@options["remote"]
+        raise ArgumentError, "You must specify both local and remote builder config for remote multiarch builds"
+      end
+
+      if @options["cache"] && @options["cache"]["type"]
+        raise ArgumentError, "Invalid cache type: #{@options["cache"]["type"]}" unless ["gha", "registry"].include?(@options["cache"]["type"])
+      end
+    end
+
+    def cache_image
+      @options["cache"]&.fetch("image", nil) || "#{@image}-build-cache"
+    end
+
+    def cache_from_config_for_gha
+      "type=gha"
+    end
+
+    def cache_from_config_for_registry
+      [ "type=registry", "ref=#{@server}/#{cache_image}" ].compact.join(",")
+    end
+
+    def cache_to_config_for_gha
+      [ "type=gha", @options["cache"]&.fetch("options", nil)].compact.join(",")
+    end
+
+    def cache_to_config_for_registry
+      [ "type=registry", @options["cache"]&.fetch("options", nil), "ref=#{@server}/#{cache_image}" ].compact.join(",")
+    end
+end