health: Adds new prometheus metrics and metrics endpoint (ory#827)

Signed-off-by: Dmitry Dolbik <[email protected]>

Gopkg.toml

@@ -144,3 +144,7 @@
 [[constraint]]
   branch = "v2"
   name = "gopkg.in/yaml.v2"
+
+[[constraint]]
+  name = "github.com/prometheus/client_golang"
+  version = "0.8.0"

cmd/server/handler.go

@@ -105,12 +105,14 @@ func RunHost(c *config.Config) func(cmd *cobra.Command, args []string) {
 		n := negroni.New()
 
 		if ok, _ := cmd.Flags().GetBool("disable-telemetry"); !ok && os.Getenv("DISABLE_TELEMETRY") != "1" {
-			metrics := c.GetMetrics()
-			go metrics.RegisterSegment()
-			go metrics.CommitMemoryStatistics()
-			n.Use(metrics)
+			telemetryMetrics := c.GetTelemetryMetrics()
+			go telemetryMetrics.RegisterSegment()
+			go telemetryMetrics.CommitMemoryStatistics()
+			n.Use(telemetryMetrics)
 		}
 
+		n.Use(c.GetPrometheusMetrics())
+
 		n.Use(negronilogrus.NewMiddlewareFromLogger(logger, c.Issuer))
 		n.UseFunc(serverHandler.rejectInsecureRequests)
 		n.UseHandler(router)

cmd/server/handler_health_factory.go

@@ -29,7 +29,7 @@ import (
 
 func newHealthHandler(c *config.Config, router *httprouter.Router) *health.Handler {
 	h := &health.Handler{
-		Metrics:        c.GetMetrics(),
+		Metrics:        c.GetTelemetryMetrics(),
 		H:              herodot.NewJSONWriter(c.GetLogger()),
 		W:              c.Context().Warden,
 		ResourcePrefix: c.AccessControlResourcePrefix,

config/config.go

@@ -37,7 +37,8 @@ import (
 	foauth2 "github.com/ory/fosite/handler/oauth2"
 	"github.com/ory/fosite/token/hmac"
 	"github.com/ory/hydra/health"
-	"github.com/ory/hydra/metrics"
+	"github.com/ory/hydra/metrics/prometheus"
+	"github.com/ory/hydra/metrics/telemetry"
 	"github.com/ory/hydra/pkg"
 	"github.com/ory/hydra/warden/group"
 	"github.com/ory/ladon"
@@ -84,15 +85,16 @@ type Config struct {
 	SendOAuth2DebugMessagesToClients bool   `mapstructure:"OAUTH2_SHARE_ERROR_DEBUG" yaml:"-"`
 	ForceHTTP                        bool   `yaml:"-"`
 
-	BuildVersion string                  `yaml:"-"`
-	BuildHash    string                  `yaml:"-"`
-	BuildTime    string                  `yaml:"-"`
-	logger       *logrus.Logger          `yaml:"-"`
-	metrics      *metrics.MetricsManager `yaml:"-"`
-	cluster      *url.URL                `yaml:"-"`
-	oauth2Client *http.Client            `yaml:"-"`
-	context      *Context                `yaml:"-"`
-	systemSecret []byte                  `yaml:"-"`
+	BuildVersion string                     `yaml:"-"`
+	BuildHash    string                     `yaml:"-"`
+	BuildTime    string                     `yaml:"-"`
+	logger       *logrus.Logger             `yaml:"-"`
+	telemetry    *telemetry.MetricsManager  `yaml:"-"`
+	prometheus   *prometheus.MetricsManager `yaml:"-"`
+	cluster      *url.URL                   `yaml:"-"`
+	oauth2Client *http.Client               `yaml:"-"`
+	context      *Context                   `yaml:"-"`
+	systemSecret []byte                     `yaml:"-"`
 }
 
 func (c *Config) GetClusterURLWithoutTailingSlash() string {
@@ -154,12 +156,21 @@ func (c *Config) GetLogger() *logrus.Logger {
 	return c.logger
 }
 
-func (c *Config) GetMetrics() *metrics.MetricsManager {
-	if c.metrics == nil {
-		c.metrics = metrics.NewMetricsManager(c.Issuer, c.DatabaseURL, c.GetLogger(), c.BuildVersion, c.BuildHash, c.BuildTime)
+func (c *Config) GetTelemetryMetrics() *telemetry.MetricsManager {
+	if c.telemetry == nil {
+		c.telemetry = telemetry.NewMetricsManager(c.Issuer, c.DatabaseURL, c.GetLogger(), c.BuildVersion, c.BuildHash, c.BuildTime)
 	}
 
-	return c.metrics
+	return c.telemetry
+}
+
+func (c *Config) GetPrometheusMetrics() *prometheus.MetricsManager {
+	if c.prometheus == nil {
+		c.GetLogger().Info("Setting up Prometheus metrics")
+		c.prometheus = prometheus.NewMetricsManager(c.BuildVersion, c.BuildHash, c.BuildTime)
+	}
+
+	return c.prometheus
 }
 
 func (c *Config) DoesRequestSatisfyTermination(r *http.Request) error {

health/handler.go

@@ -26,15 +26,17 @@ import (
 	"github.com/julienschmidt/httprouter"
 	"github.com/ory/herodot"
 	"github.com/ory/hydra/firewall"
-	"github.com/ory/hydra/metrics"
+	"github.com/ory/hydra/metrics/telemetry"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 const (
-	HealthStatusPath = "/health/status"
+	HealthStatusPath     = "/health/status"
+	PrometheusStatusPath = "/health/prometheus"
 )
 
 type Handler struct {
-	Metrics        *metrics.MetricsManager
+	Metrics        *telemetry.MetricsManager
 	H              *herodot.JSONWriter
 	W              firewall.Firewall
 	ResourcePrefix string
@@ -54,6 +56,7 @@ func (h *Handler) PrefixResource(resource string) string {
 
 func (h *Handler) SetRoutes(r *httprouter.Router) {
 	r.GET(HealthStatusPath, h.Health)
+	r.Handler("GET", PrometheusStatusPath, promhttp.Handler())
 }
 
 // swagger:route GET /health/status health getInstanceStatus

metrics/prometheus/metrics.go

@@ -0,0 +1,42 @@
+package prometheus
+
+// Metrics prototypes
+// Example:
+// 	Counter      *prometheus.CounterVec
+// 	ResponseTime *prometheus.HistogramVec
+type Metrics struct{}
+
+// Method for creation new custom Prometheus  metrics
+// Example:
+// 	pm := &Metrics{
+//		Counter: prometheus.NewCounterVec(
+//			prometheus.CounterOpts{
+//				Name:        "servicename_requests_total",
+//				Help:        "Description",
+//				ConstLabels: map[string]string{
+//					"version":   version,
+//					"hash":      hash,
+//					"buildTime": buildTime,
+//				},
+//			},
+//			[]string{"endpoint"},
+//		),
+//		ResponseTime: prometheus.NewHistogramVec(
+//			prometheus.HistogramOpts{
+//				Name:        "servicename_response_time_seconds",
+//				Help:        "Description",
+//				ConstLabels: map[string]string{
+//					"version":   version,
+//					"hash":      hash,
+//					"buildTime": buildTime,
+//				},
+//			},
+//			[]string{"endpoint"},
+//		),
+//	}
+//	prometheus.Register(pm.Counter)
+//  prometheus.Register(pm.ResponseTime)
+func NewMetrics(version, hash, buildTime string) *Metrics {
+	pm := &Metrics{}
+	return pm
+}

metrics/prometheus/middleware.go

@@ -0,0 +1,27 @@
+package prometheus
+
+import (
+	"net/http"
+)
+
+type MetricsManager struct {
+	prometheusMetrics *Metrics
+}
+
+func NewMetricsManager(version, hash, buildTime string) *MetricsManager {
+	return &MetricsManager{
+		prometheusMetrics: NewMetrics(version, hash, buildTime),
+	}
+}
+
+// Main middleware method to collect metrics for Prometheus.
+// Example:
+//	start := time.Now()
+//	next(rw, r)
+//	Request counter metric
+//	pmm.prometheusMetrics.Counter.WithLabelValues(r.URL.Path).Inc()
+//	Response time metric
+//	pmm.prometheusMetrics.ResponseTime.WithLabelValues(r.URL.Path).Observe(time.Since(start).Seconds())
+func (pmm *MetricsManager) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
+	next(rw, r)
+}

metrics/metrics.go → metrics/telemetry/metrics.go

@@ -18,7 +18,7 @@
  * @license 	Apache-2.0
  */
 
-package metrics
+package telemetry
 
 import (
 	"runtime"

metrics/middleware.go → metrics/telemetry/middleware.go

@@ -18,7 +18,7 @@
  * @license 	Apache-2.0
  */
 
-package metrics
+package telemetry
 
 import (
 	"crypto/sha256"
@@ -100,6 +100,7 @@ func NewMetricsManager(issuerURL string, databaseURL string, l logrus.FieldLogge
 		salt:         uuid.New(),
 		BuildTime:    buildTime, BuildVersion: version, BuildHash: hash,
 	}
+
 	return mm
 }
 

metrics/middleware_test.go → metrics/telemetry/middleware_test.go

@@ -18,7 +18,7 @@
  * @license 	Apache-2.0
  */
 
-package metrics
+package telemetry
 
 import (
 	"net/url"