ocsp-updater: add support for writing to Redis (letsencrypt#5825)

If configured, ocsp-updater will write responses to Redis in parallel
with MariaDB, giving up if Redis is slower and incrementing a stat.

Factors out the ShortIDIssuer concept from rocsp-tool into
rocsp_config.

cmd/ocsp-updater/main.go

@@ -16,6 +16,8 @@ import (
 	bgrpc "github.com/letsencrypt/boulder/grpc"
 	"github.com/letsencrypt/boulder/ocsp_updater"
 	ocsp_updater_config "github.com/letsencrypt/boulder/ocsp_updater/config"
+	"github.com/letsencrypt/boulder/rocsp"
+	rocsp_config "github.com/letsencrypt/boulder/rocsp/config"
 	"github.com/letsencrypt/boulder/sa"
 )
 
@@ -112,6 +114,15 @@ func main() {
 
 	clk := cmd.Clock()
 
+	redisConf := c.OCSPUpdater.Redis
+	var rocspClient *rocsp.WritingClient
+	if redisConf != nil {
+		rocspClient, err = rocsp_config.MakeClient(redisConf, clk)
+		cmd.FailOnError(err, "making Redis client")
+	}
+	issuers, err := rocsp_config.LoadIssuers(c.OCSPUpdater.Issuers)
+	cmd.FailOnError(err, "loading issuers")
+
 	tlsConfig, err := c.OCSPUpdater.TLS.Load()
 	cmd.FailOnError(err, "TLS config")
 	clientMetrics := bgrpc.NewClientMetrics(stats)
@@ -129,6 +140,8 @@ func main() {
 		clk,
 		db,
 		readOnlyDb,
+		rocspClient,
+		issuers,
 		serialSuffixes,
 		ogc,
 		// Necessary evil for now

cmd/rocsp-tool/client.go

@@ -15,13 +15,14 @@ import (
 	capb "github.com/letsencrypt/boulder/ca/proto"
 	"github.com/letsencrypt/boulder/core"
 	"github.com/letsencrypt/boulder/rocsp"
+	rocsp_config "github.com/letsencrypt/boulder/rocsp/config"
 	"github.com/letsencrypt/boulder/sa"
 	"github.com/letsencrypt/boulder/test/ocsp/helper"
 	"golang.org/x/crypto/ocsp"
 )
 
 type client struct {
-	issuers       []shortIDIssuer
+	issuers       []rocsp_config.ShortIDIssuer
 	redis         *rocsp.WritingClient
 	db            *sql.DB // optional
 	ocspGenerator capb.OCSPGeneratorClient
@@ -218,13 +219,13 @@ func (cl *client) signAndStoreResponses(ctx context.Context, input <-chan *sa.Ce
 		}
 		// ttl is the lifetime of the certificate
 		ttl := cl.clk.Now().Sub(status.NotAfter)
-		issuer, err := findIssuerByID(status.IssuerID, cl.issuers)
+		issuer, err := rocsp_config.FindIssuerByID(status.IssuerID, cl.issuers)
 		if err != nil {
 			output <- processResult{id: uint64(status.ID), err: err}
 			continue
 		}
 
-		err = cl.redis.StoreResponse(ctx, result.Response, issuer.shortID, ttl)
+		err = cl.redis.StoreResponse(ctx, result.Response, issuer.ShortID(), ttl)
 		if err != nil {
 			output <- processResult{id: uint64(status.ID), err: err}
 		} else {
@@ -261,7 +262,7 @@ func (cl *client) storeResponse(ctx context.Context, respBytes []byte, ttl *time
 	if err != nil {
 		return fmt.Errorf("parsing response: %w", err)
 	}
-	issuer, err := findIssuerByName(resp, cl.issuers)
+	issuer, err := rocsp_config.FindIssuerByName(resp, cl.issuers)
 	if err != nil {
 		return fmt.Errorf("finding issuer for response: %w", err)
 	}
@@ -297,7 +298,7 @@ func (cl *client) storeResponse(ctx context.Context, respBytes []byte, ttl *time
 		ttl.Hours(),
 	)
 
-	err = cl.redis.StoreResponse(ctx, respBytes, issuer.shortID, *ttl)
+	err = cl.redis.StoreResponse(ctx, respBytes, issuer.ShortID(), *ttl)
 	if err != nil {
 		return fmt.Errorf("storing response: %w", err)
 	}

cmd/rocsp-tool/client_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/letsencrypt/boulder/core"
 	"github.com/letsencrypt/boulder/log"
 	"github.com/letsencrypt/boulder/rocsp"
+	rocsp_config "github.com/letsencrypt/boulder/rocsp/config"
 	"github.com/letsencrypt/boulder/sa"
 	"github.com/letsencrypt/boulder/test"
 	"github.com/letsencrypt/boulder/test/vars"
@@ -90,7 +91,7 @@ func TestStoreResponse(t *testing.T) {
 	}, issuerKey)
 	test.AssertNotError(t, err, "creating OCSP response")
 
-	issuers, err := loadIssuers(map[string]int{
+	issuers, err := rocsp_config.LoadIssuers(map[string]int{
 		"../../test/hierarchy/int-e1.cert.pem": 23,
 	})
 	if err != nil {

cmd/rocsp-tool/main.go

@@ -90,7 +90,7 @@ func main2() error {
 		return fmt.Errorf("reading JSON config file: %w", err)
 	}
 
-	issuers, err := loadIssuers(c.ROCSPTool.Issuers)
+	issuers, err := rocsp_config.LoadIssuers(c.ROCSPTool.Issuers)
 	if err != nil {
 		return fmt.Errorf("loading issuers: %w", err)
 	}

ocsp_updater/config/ocsp_updater_config.go

@@ -1,13 +1,24 @@
 package ocsp_updater_config
 
-import "github.com/letsencrypt/boulder/cmd"
+import (
+	"github.com/letsencrypt/boulder/cmd"
+	rocsp_config "github.com/letsencrypt/boulder/rocsp/config"
+)
 
 // Config provides the various window tick times and batch sizes needed
 // for the OCSP updater
 type Config struct {
 	cmd.ServiceConfig
 	DB         cmd.DBConfig
 	ReadOnlyDB cmd.DBConfig
+	Redis      *rocsp_config.RedisConfig
+
+	// Issuers is a map from filenames to short issuer IDs.
+	// Each filename must contain an issuer certificate. The short issuer
+	// IDs are arbitrarily assigned and must be consistent across OCSP
+	// components. For production we'll use the number part of the CN, i.e.
+	// E1 -> 1, R3 -> 3, etc.
+	Issuers map[string]int
 
 	OldOCSPWindow    cmd.ConfigDuration
 	OldOCSPBatchSize int

ocsp_updater/updater.go

@@ -16,6 +16,7 @@ import (
 	"github.com/letsencrypt/boulder/core"
 	blog "github.com/letsencrypt/boulder/log"
 	ocsp_updater_config "github.com/letsencrypt/boulder/ocsp_updater/config"
+	rocsp_config "github.com/letsencrypt/boulder/rocsp/config"
 	"github.com/letsencrypt/boulder/sa"
 )
 
@@ -34,6 +35,10 @@ type ocspDb interface {
 	Exec(query string, args ...interface{}) (sql.Result, error)
 }
 
+type rocspClientInterface interface {
+	StoreResponse(ctx context.Context, respBytes []byte, shortIssuerID byte, ttl time.Duration) error
+}
+
 // failCounter provides a concurrent safe counter.
 type failCounter struct {
 	mu    sync.Mutex
@@ -63,8 +68,11 @@ type OCSPUpdater struct {
 	log blog.Logger
 	clk clock.Clock
 
-	db         ocspDb
-	readOnlyDb ocspReadOnlyDb
+	db          ocspDb
+	readOnlyDb  ocspReadOnlyDb
+	rocspClient rocspClientInterface
+
+	issuers []rocsp_config.ShortIDIssuer
 
 	ogc capb.OCSPGeneratorClient
 
@@ -89,6 +97,7 @@ type OCSPUpdater struct {
 	genStoreHistogram    prometheus.Histogram
 	generatedCounter     *prometheus.CounterVec
 	storedCounter        *prometheus.CounterVec
+	storedRedisCounter   *prometheus.CounterVec
 	markExpiredCounter   *prometheus.CounterVec
 	findStaleOCSPCounter *prometheus.CounterVec
 }
@@ -98,6 +107,8 @@ func New(
 	clk clock.Clock,
 	db ocspDb,
 	readOnlyDb ocspReadOnlyDb,
+	rocspClient rocspClientInterface,
+	issuers []rocsp_config.ShortIDIssuer,
 	serialSuffixes []string,
 	ogc capb.OCSPGeneratorClient,
 	config ocsp_updater_config.Config,
@@ -142,6 +153,10 @@ func New(
 		Help: "A counter of OCSP response generation calls labeled by result",
 	}, []string{"result"})
 	stats.MustRegister(generatedCounter)
+	storedRedisCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "ocsp_updater_stored_redis",
+		Help: "A counter of OCSP response storage calls labeled by result",
+	}, []string{"result"})
 	storedCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
 		Name: "ocsp_updater_stored",
 		Help: "A counter of OCSP response storage calls labeled by result",
@@ -170,17 +185,24 @@ func New(
 	}, []string{"result"})
 	stats.MustRegister(findStaleOCSPCounter)
 
+	var rocspClientInterface rocspClientInterface
+	if rocspClient != nil {
+		rocspClientInterface = rocspClient
+	}
 	updater := OCSPUpdater{
 		clk:                          clk,
 		db:                           db,
 		readOnlyDb:                   readOnlyDb,
+		rocspClient:                  rocspClientInterface,
+		issuers:                      issuers,
 		ogc:                          ogc,
 		log:                          log,
 		ocspMinTimeToExpiry:          config.OCSPMinTimeToExpiry.Duration,
 		parallelGenerateOCSPRequests: config.ParallelGenerateOCSPRequests,
 		genStoreHistogram:            genStoreHistogram,
 		generatedCounter:             generatedCounter,
 		storedCounter:                storedCounter,
+		storedRedisCounter:           storedRedisCounter,
 		markExpiredCounter:           markExpiredCounter,
 		findStaleOCSPCounter:         findStaleOCSPCounter,
 		stalenessHistogram:           stalenessHistogram,
@@ -297,7 +319,30 @@ func (updater *OCSPUpdater) generateResponse(ctx context.Context, status sa.Cert
 }
 
 // storeResponse stores a given CertificateStatus in the database.
-func (updater *OCSPUpdater) storeResponse(status *sa.CertStatusMetadata) error {
+func (updater *OCSPUpdater) storeResponse(ctx context.Context, status *sa.CertStatusMetadata) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	if updater.rocspClient != nil {
+		go func() {
+			ttl := status.NotAfter.Sub(updater.clk.Now())
+			shortIssuerID, err := rocsp_config.FindIssuerByID(status.IssuerID, updater.issuers)
+			if err != nil {
+				updater.storedRedisCounter.WithLabelValues("missing issuer").Inc()
+				return
+			}
+			err = updater.rocspClient.StoreResponse(ctx, status.OCSPResponse, shortIssuerID.ShortID(), ttl)
+			if err != nil {
+				if errors.Is(err, context.Canceled) {
+					updater.storedRedisCounter.WithLabelValues("canceled").Inc()
+				} else {
+					updater.storedRedisCounter.WithLabelValues("failed").Inc()
+				}
+			} else {
+				updater.storedRedisCounter.WithLabelValues("success").Inc()
+			}
+		}()
+	}
 	// Update the certificateStatus table with the new OCSP response, the status
 	// WHERE is used make sure we don't overwrite a revoked response with a one
 	// containing a 'good' status.
@@ -311,6 +356,12 @@ func (updater *OCSPUpdater) storeResponse(status *sa.CertStatusMetadata) error {
 		status.ID,
 		string(status.Status),
 	)
+
+	if err != nil {
+		updater.storedCounter.WithLabelValues("failed").Inc()
+	} else {
+		updater.storedCounter.WithLabelValues("success").Inc()
+	}
 	return err
 }
 
@@ -385,7 +436,7 @@ func (updater *OCSPUpdater) generateOCSPResponses(ctx context.Context, staleStat
 		}
 		updater.generatedCounter.WithLabelValues("success").Inc()
 
-		err = updater.storeResponse(meta)
+		err = updater.storeResponse(ctx, meta)
 		if err != nil {
 			updater.log.AuditErrf("Failed to store OCSP response: %s", err)
 			updater.storedCounter.WithLabelValues("failed").Inc()