Add log validator to integration tests (letsencrypt#4782)

For now this mainly provides an example config and confirms that log-validator can start up and shut down cleanly, as well as provide a stat indicating how many log lines it has handled. This introduces a syslog config to the boulder-tools image that will write logs to /var/log/program.log. It also tweaks the various .json config files so they have non-default syslogLevel, to ensure they actually write something for log-validator to verify.
wthayer-fastly · Apr 20, 2020 · 87fb602 · 87fb602
1 parent 8259925
commit 87fb602
Show file tree

Hide file tree

Showing 41 changed files with 114 additions and 45 deletions.
diff --git a/cmd/log-validator/main.go b/cmd/log-validator/main.go
@@ -127,11 +127,11 @@ func main() {
 	cmd.FailOnError(err, "failed to parse config file")
 
 	stats, logger := cmd.StatsAndLogging(config.Syslog, config.DebugAddr)
-	badLineCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
-		Name: "bad_log_lines",
-		Help: "A counter of corrupt log lines",
-	}, []string{"filename"})
-	stats.MustRegister(badLineCounter)
+	lineCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "log_lines",
+		Help: "A counter of log lines processed, with status",
+	}, []string{"filename", "status"})
+	stats.MustRegister(lineCounter)
 
 	var tailers []*tail.Tail
 	for _, filename := range config.Files {
@@ -151,8 +151,10 @@ func main() {
 					continue
 				}
 				if err := lineValid(line.Text); err != nil {
-					badLineCounter.WithLabelValues(t.Filename).Inc()
+					lineCounter.WithLabelValues(t.Filename, "bad").Inc()
 					logger.Errf("%s: %s %q", t.Filename, err, line.Text)
+				} else {
+					lineCounter.WithLabelValues(t.Filename, "ok").Inc()
 				}
 			}
 		}()

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -2,7 +2,7 @@ version: '3'
 services:
     boulder:
         # To minimize fetching this should be the same version used below
-        image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-08
+        image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-16
         environment:
             FAKE_DNS: 10.77.77.77
             PKCS11_PROXY_SOCKET: tcp://boulder-hsm:5657
@@ -65,7 +65,7 @@ services:
             hard: -1
     bhsm:
         # To minimize fetching this should be the same version used above
-        image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-08
+        image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-16
         environment:
             PKCS11_DAEMON_SOCKET: tcp://0.0.0.0:5657
         command: /usr/local/bin/pkcs11-daemon /usr/lib/softhsm/libsofthsm2.so
@@ -92,7 +92,7 @@ services:
         logging:
             driver: none
     netaccess:
-        image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-08
+        image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-04-16
         environment:
             GO111MODULE: "on"
             GOFLAGS: "-mod=vendor"

diff --git a/test/boulder-tools/Dockerfile.tmpl b/test/boulder-tools/Dockerfile.tmpl
@@ -9,6 +9,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
 WORKDIR $GOPATH
 
 RUN wget -O go.tgz "https://dl.google.com/go/go${GO_VERSION_TO_INSTALL}.linux-amd64.tar.gz" && tar -C /usr/local -xzf go.tgz && rm go.tgz;
-ADD requirements.txt /tmp/requirements.txt
-ADD build.sh /tmp/build.sh
+COPY requirements.txt /tmp/requirements.txt
+COPY boulder.rsyslog.conf /etc/rsyslog.d/
+COPY build.sh /tmp/build.sh
 RUN /tmp/build.sh
diff --git a/test/boulder-tools/boulder.rsyslog.conf b/test/boulder-tools/boulder.rsyslog.conf
@@ -0,0 +1,18 @@
+module( load="builtin:omfile" template="LELogFormat" )
+
+template( name="LELogFormat" type="list" ) {
+  property(name="timereported" dateFormat="rfc3339")
+  constant(value=" ")
+  property(name="hostname" field.delimiter="46" field.number="1")
+  constant(value=" datacenter ")
+  property(name="syslogseverity")
+  constant(value=" ")
+  property(name="syslogtag")
+  property(name="msg" spifno1stsp="on" )
+  property(name="msg" droplastlf="on" )
+  constant(value="\n")
+}
+
+template( name="TmplAll" type="string" string="/var/log/%PROGRAMNAME%.log" )
+
+action( type="omfile" dynaFile="TmplAll" )
diff --git a/test/config-next/admin-revoker.json b/test/config-next/admin-revoker.json
@@ -21,6 +21,6 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   }
 }
diff --git a/test/config-next/akamai-purger.json b/test/config-next/akamai-purger.json
@@ -22,11 +22,11 @@
 
     "syslog": {
       "stdoutlevel": 6,
-      "sysloglevel": 4
+      "sysloglevel": 6
     },
 
     "common": {
       "issuerCert": "test/test-ca2.pem"
     }
   }
-
+
diff --git a/test/config-next/ca-a.json b/test/config-next/ca-a.json
@@ -152,6 +152,6 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   }
 }
diff --git a/test/config-next/ca-b.json b/test/config-next/ca-b.json
@@ -153,6 +153,6 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   }
 }
diff --git a/test/config-next/cert-checker.json b/test/config-next/cert-checker.json
@@ -18,6 +18,6 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   }
 }
diff --git a/test/config-next/expiration-mailer.json b/test/config-next/expiration-mailer.json
@@ -27,6 +27,6 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   }
 }
diff --git a/test/config-next/log-validator.json b/test/config-next/log-validator.json
@@ -0,0 +1,19 @@
+{
+  "syslog": {
+    "stdoutLevel": 7
+  },
+  "debugAddr": ":8016",
+  "files": [
+    "/var/log/akamai-purger.log",
+    "/var/log/boulder-ca.log",
+    "/var/log/boulder-publisher.log",
+    "/var/log/boulder-ra.log",
+    "/var/log/boulder-sa.log",
+    "/var/log/boulder-va.log",
+    "/var/log/boulder-wfe2.log",
+    "/var/log/boulder-wfe.log",
+    "/var/log/nonce-service.log",
+    "/var/log/ocsp-responder.log",
+    "/var/log/ocsp-updater.log"
+  ]
+}
diff --git a/test/config-next/nonce.json b/test/config-next/nonce.json
@@ -3,7 +3,8 @@
         "maxUsed": 131072,
         "noncePrefix": "taro",
         "syslog": {
-            "stdoutLevel": 6
+            "stdoutLevel": 6,
+            "syslogLevel": 6
         },
         "debugAddr": ":8111",
         "grpc": {

diff --git a/test/config-next/notify-mailer.json b/test/config-next/notify-mailer.json
@@ -6,5 +6,9 @@
     "passwordFile": "test/secrets/smtp_password",
     "dbConnectFile": "test/secrets/mailer_dburl",
     "maxDBConns": 10
-  }
+  },
+   "syslog": {
+    "stdoutLevel": 7,
+    "syslogLevel": 7
+   }
 }
diff --git a/test/config-next/ocsp-responder.json b/test/config-next/ocsp-responder.json
@@ -13,7 +13,7 @@
 
   "syslog": {
    "stdoutlevel": 6,
-   "sysloglevel": 4
+   "sysloglevel": 6
  },
 
   "common": {

diff --git a/test/config-next/ocsp-updater.json b/test/config-next/ocsp-updater.json
@@ -33,7 +33,7 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   },
 
   "common": {

diff --git a/test/config-next/orphan-finder.json b/test/config-next/orphan-finder.json
@@ -2,6 +2,7 @@
   "backdate": "1h",
 
   "syslog": {
+    "stdoutlevel": 7,
     "stdoutlevel": 7
   },
 

diff --git a/test/config-next/publisher.json b/test/config-next/publisher.json
@@ -23,7 +23,7 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   },
 
   "common": {

diff --git a/test/config-next/ra.json b/test/config-next/ra.json
@@ -120,6 +120,6 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   }
 }
diff --git a/test/config-next/sa.json b/test/config-next/sa.json
@@ -31,6 +31,6 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   }
 }
diff --git a/test/config-next/va.json b/test/config-next/va.json
@@ -50,7 +50,7 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   },
 
   "common": {

diff --git a/test/config-next/wfe.json b/test/config-next/wfe.json
@@ -46,7 +46,7 @@
 
   "syslog": {
     "stdoutlevel": 4,
-    "sysloglevel": 4
+    "sysloglevel": 6
   },
 
   "common": {

diff --git a/test/config-next/wfe2.json b/test/config-next/wfe2.json
@@ -60,7 +60,7 @@
 
   "syslog": {
     "stdoutlevel": 4,
-    "sysloglevel": 4
+    "sysloglevel": 6
   },
 
   "common": {

diff --git a/test/config/admin-revoker.json b/test/config/admin-revoker.json
@@ -21,6 +21,6 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   }
 }
diff --git a/test/config/akamai-purger.json b/test/config/akamai-purger.json
@@ -22,11 +22,11 @@
 
     "syslog": {
       "stdoutlevel": 6,
-      "sysloglevel": 4
+      "sysloglevel": 6
     },
 
     "common": {
       "issuerCert": "test/test-ca2.pem"
     }
   }
-
+
diff --git a/test/config/ca-a.json b/test/config/ca-a.json
@@ -152,6 +152,6 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   }
 }
diff --git a/test/config/ca-b.json b/test/config/ca-b.json
@@ -153,6 +153,6 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   }
 }
diff --git a/test/config/cert-checker.json b/test/config/cert-checker.json
@@ -18,6 +18,6 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   }
 }
diff --git a/test/config/expiration-mailer.json b/test/config/expiration-mailer.json
@@ -27,6 +27,6 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   }
 }
diff --git a/test/config/log-validator.json b/test/config/log-validator.json
@@ -0,0 +1,19 @@
+{
+  "syslog": {
+    "stdoutLevel": 7
+  },
+  "debugAddr": ":8016",
+  "files": [
+    "/var/log/akamai-purger.log",
+    "/var/log/boulder-ca.log",
+    "/var/log/boulder-publisher.log",
+    "/var/log/boulder-ra.log",
+    "/var/log/boulder-sa.log",
+    "/var/log/boulder-va.log",
+    "/var/log/boulder-wfe2.log",
+    "/var/log/boulder-wfe.log",
+    "/var/log/nonce-service.log",
+    "/var/log/ocsp-responder.log",
+    "/var/log/ocsp-updater.log"
+  ]
+}
diff --git a/test/config/nonce.json b/test/config/nonce.json
@@ -3,7 +3,8 @@
         "maxUsed": 131072,
         "noncePrefix": "taro",
         "syslog": {
-            "stdoutLevel": 6
+            "stdoutLevel": 6,
+            "syslogLevel": 6
         },
         "debugAddr": ":8111",
         "grpc": {

diff --git a/test/config/notify-mailer.json b/test/config/notify-mailer.json
@@ -8,6 +8,7 @@
     "maxDBConns": 10
   },
   "syslog": {
-    "stdoutLevel": 7
+    "stdoutLevel": 7,
+    "syslogLevel": 7
   }
 }
diff --git a/test/config/ocsp-responder.json b/test/config/ocsp-responder.json
@@ -11,7 +11,7 @@
 
   "syslog": {
    "stdoutlevel": 6,
-   "sysloglevel": 4
+   "sysloglevel": 6
  },
 
   "common": {

diff --git a/test/config/ocsp-updater.json b/test/config/ocsp-updater.json
@@ -32,7 +32,7 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   },
 
   "common": {

diff --git a/test/config/orphan-finder.json b/test/config/orphan-finder.json
@@ -2,6 +2,7 @@
   "backdate": "1h",
 
   "syslog": {
+    "stdoutlevel": 7,
     "stdoutlevel": 7
   },
 

diff --git a/test/config/publisher.json b/test/config/publisher.json
@@ -22,7 +22,7 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   },
 
   "common": {

diff --git a/test/config/ra.json b/test/config/ra.json
@@ -118,6 +118,6 @@
 
   "syslog": {
     "stdoutlevel": 6,
-    "sysloglevel": 4
+    "sysloglevel": 6
   }
 }