SIGUSR2 triggers graceful binary upgrades (spawns new process) (caddy…

…server#1814)

* SIGUSR2 triggers graceful binary upgrades (spawns new process)

* Move some functions around, hopefully fixing Windows build

* Clean up a couple file closes and add links to useful debugging thread

* Use two underscores in upgrade env var

To help ensure uniqueness / avoid possible collisions

caddy.go

@@ -17,6 +17,7 @@ package caddy
 
 import (
 	"bytes"
+	"encoding/gob"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -51,7 +52,7 @@ var (
 	// isUpgrade will be set to true if this process
 	// was started as part of an upgrade, where a parent
 	// Caddy process started this one.
-	isUpgrade bool
+	isUpgrade = os.Getenv("CADDY__UPGRADE") == "1"
 
 	// started will be set to true when the first
 	// instance is started; it never gets set to
@@ -360,6 +361,16 @@ type AfterStartup interface {
 // is returned. Consequently, this function never returns a nil
 // value as long as there are no errors.
 func LoadCaddyfile(serverType string) (Input, error) {
+	// If we are finishing an upgrade, we must obtain the Caddyfile
+	// from our parent process, regardless of configured loaders.
+	if IsUpgrade() {
+		err := gob.NewDecoder(os.Stdin).Decode(&loadedGob)
+		if err != nil {
+			return nil, err
+		}
+		return loadedGob.Caddyfile, nil
+	}
+
 	// Ask plugged-in loaders for a Caddyfile
 	cdyfile, err := loadCaddyfileInput(serverType)
 	if err != nil {
@@ -424,9 +435,16 @@ func (i *Instance) Caddyfile() Input {
 //
 // This function blocks until all the servers are listening.
 func Start(cdyfile Input) (*Instance, error) {
-	writePidFile()
 	inst := &Instance{serverType: cdyfile.ServerType(), wg: new(sync.WaitGroup)}
-	return inst, startWithListenerFds(cdyfile, inst, nil)
+	err := startWithListenerFds(cdyfile, inst, nil)
+	if err != nil {
+		return inst, err
+	}
+	signalSuccessToParent()
+	if pidErr := writePidFile(); pidErr != nil {
+		log.Printf("[ERROR] Could not write pidfile: %v", pidErr)
+	}
+	return inst, nil
 }
 
 func startWithListenerFds(cdyfile Input, inst *Instance, restartFds map[string]restartTriple) error {
@@ -445,7 +463,8 @@ func startWithListenerFds(cdyfile Input, inst *Instance, restartFds map[string]r
 	}
 
 	// run startup callbacks
-	if restartFds == nil {
+	if !IsUpgrade() && restartFds == nil {
+		// first startup means not a restart or upgrade
 		for _, firstStartupFunc := range inst.onFirstStartup {
 			err := firstStartupFunc()
 			if err != nil {
@@ -500,7 +519,6 @@ func startWithListenerFds(cdyfile Input, inst *Instance, restartFds map[string]r
 // callbacks will not be executed between directives, since the purpose
 // is only to check the input for valid syntax.
 func ValidateAndExecuteDirectives(cdyfile Input, inst *Instance, justValidate bool) error {
-
 	// If parsing only inst will be nil, create an instance for this function call only.
 	if justValidate {
 		inst = &Instance{serverType: cdyfile.ServerType(), wg: new(sync.WaitGroup)}
@@ -536,7 +554,6 @@ func ValidateAndExecuteDirectives(cdyfile Input, inst *Instance, justValidate bo
 	}
 
 	return nil
-
 }
 
 func executeDirectives(inst *Instance, filename string,
@@ -616,6 +633,30 @@ func startServers(serverList []Server, inst *Instance, restartFds map[string]res
 			err error
 		)
 
+		// if performing an upgrade, obtain listener file descriptors
+		// from parent process
+		if IsUpgrade() {
+			if gs, ok := s.(GracefulServer); ok {
+				addr := gs.Address()
+				if fdIndex, ok := loadedGob.ListenerFds["tcp"+addr]; ok {
+					file := os.NewFile(fdIndex, "")
+					ln, err = net.FileListener(file)
+					file.Close()
+					if err != nil {
+						return err
+					}
+				}
+				if fdIndex, ok := loadedGob.ListenerFds["udp"+addr]; ok {
+					file := os.NewFile(fdIndex, "")
+					pc, err = net.FilePacketConn(file)
+					file.Close()
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+
 		// If this is a reload and s is a GracefulServer,
 		// reuse the listener for a graceful restart.
 		if gs, ok := s.(GracefulServer); ok && restartFds != nil {
@@ -795,25 +836,6 @@ func IsInternal(addr string) bool {
 	return false
 }
 
-// Upgrade re-launches the process, preserving the listeners
-// for a graceful restart. It does NOT load new configuration;
-// it only starts the process anew with a fresh binary.
-//
-// TODO: This is not yet implemented
-func Upgrade() error {
-	return fmt.Errorf("not implemented")
-	// TODO: have child process set isUpgrade = true
-}
-
-// IsUpgrade returns true if this process is part of an upgrade
-// where a parent caddy process spawned this one to upgrade
-// the binary.
-func IsUpgrade() bool {
-	mu.Lock()
-	defer mu.Unlock()
-	return isUpgrade
-}
-
 // Started returns true if at least one instance has been
 // started by this package. It never gets reset to false
 // once it is set to true.

caddyhttp/httpserver/https.go

@@ -44,9 +44,14 @@ func activateHTTPS(cctx caddy.Context) error {
 	// renew all relevant certificates that need renewal. this is important
 	// to do right away so we guarantee that renewals aren't missed, and
 	// also the user can respond to any potential errors that occur.
-	err = caddytls.RenewManagedCertificates(true)
-	if err != nil {
-		return err
+	// (skip if upgrading, because the parent process is likely already listening
+	// on the ports we'd need to do ACME before we finish starting; parent process
+	// already running renewal ticker, so renewal won't be missed anyway.)
+	if !caddy.IsUpgrade() {
+		err = caddytls.RenewManagedCertificates(true)
+		if err != nil {
+			return err
+		}
 	}
 
 	if !caddy.Quiet && operatorPresent {

sigtrap_posix.go

@@ -13,7 +13,7 @@ import (
 func trapSignalsPosix() {
 	go func() {
 		sigchan := make(chan os.Signal, 1)
-		signal.Notify(sigchan, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGUSR1)
+		signal.Notify(sigchan, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGUSR1, syscall.SIGUSR2)
 
 		for sig := range sigchan {
 			switch sig {
@@ -48,19 +48,9 @@ func trapSignalsPosix() {
 				log.Println("[INFO] SIGUSR1: Reloading")
 
 				// Start with the existing Caddyfile
-				instancesMu.Lock()
-				if len(instances) == 0 {
-					instancesMu.Unlock()
-					log.Println("[ERROR] SIGUSR1: No server instances are fully running")
-					continue
-				}
-				inst := instances[0] // we only support one instance at this time
-				instancesMu.Unlock()
-
-				updatedCaddyfile := inst.caddyfileInput
-				if updatedCaddyfile == nil {
-					// Hmm, did spawing process forget to close stdin? Anyhow, this is unusual.
-					log.Println("[ERROR] SIGUSR1: no Caddyfile to reload (was stdin left open?)")
+				caddyfileToUse, inst, err := getCurrentCaddyfile()
+				if err != nil {
+					log.Printf("[ERROR] SIGUSR1: %v", err)
 					continue
 				}
 				if loaderUsed.loader == nil {
@@ -76,14 +66,20 @@ func trapSignalsPosix() {
 					continue
 				}
 				if newCaddyfile != nil {
-					updatedCaddyfile = newCaddyfile
+					caddyfileToUse = newCaddyfile
 				}
 
 				// Kick off the restart; our work is done
-				inst, err = inst.Restart(updatedCaddyfile)
+				inst, err = inst.Restart(caddyfileToUse)
 				if err != nil {
 					log.Printf("[ERROR] SIGUSR1: %v", err)
 				}
+
+			case syscall.SIGUSR2:
+				log.Println("[INFO] SIGUSR2: Upgrading")
+				if err := Upgrade(); err != nil {
+					log.Printf("[ERROR] SIGUSR2: upgrading: %v", err)
+				}
 			}
 		}
 	}()