Skip to content

Commit

Permalink
Merge pull request statping#806 from thatInfrastructureGuy/grpcCheckF…
Browse files Browse the repository at this point in the history 
…ixes

Enhance GRPC Monitoring
  • Loading branch information
@hunterlong
hunterlong authored Oct 1, 2020
2 parents 781b8db + 09b554e commit fccc559
Show file tree
Hide file tree
Showing 4 changed files with 332 additions and 12 deletions.
48 changes: 45 additions & 3 deletions frontend/src/forms/Service.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
<div class="form-group row">
<label for="service_type" class="col-sm-4 col-form-label">{{ $t('service_type') }}</label>
<div class="col-sm-8">
<select v-model="service.type" class="form-control" id="service_type">
<select v-model="service.type" @change="updateDefaultValues()" class="form-control" id="service_type">
<option value="http">HTTP {{ $t('service') }}</option>
<option value="tcp">TCP {{ $t('service') }}</option>
<option value="udp">UDP {{ $t('service') }}</option>
Expand Down Expand Up @@ -155,8 +155,7 @@
</span>
</div>
</div>

<div v-if="service.type.match(/^(http)$/)" class="form-group row">
<div v-if="service.type.match(/^(http|grpc)$/)" class="form-group row">
<label class="col-12 col-md-4 col-form-label">{{ $t('verify_ssl') }}</label>
<div class="col-12 col-md-8 mt-1 mb-2 mb-md-0">
<span @click="service.verify_ssl = !!service.verify_ssl" class="switch float-left">
Expand All @@ -167,6 +166,33 @@
</div>
</div>

<div v-if="service.type.match(/^(grpc)$/)" class="form-group row">
<label class="col-12 col-md-4 col-form-label"><a href="https://github.com/grpc/grpc/blob/master/doc/health-checking.md#grpc-health-checking-protocol">GRPC Health Check</a></label>
<div class="col-12 col-md-8 mt-1 mb-2 mb-md-0">
<span @click="service.grpc_health_check = !!service.grpc_health_check" class="switch float-left">
<input v-model="service.grpc_health_check" type="checkbox" name="grpc_health_check-option" class="switch" id="switch-grpc-health-check" v-bind:checked="service.grpc_health_check">
<label for="switch-grpc-health-check" v-if="service.grpc_health_check">Check against GRPC health check endpoint.</label>
<label for="switch-grpc-health-check" v-if="!service.grpc_health_check">Only checks if GRPC connection can be established.</label>
</span>
</div>
</div>

<div v-if="service.grpc_health_check" class="form-group row">
<label class="col-sm-4 col-form-label">Expected Response</label>
<div class="col-sm-8">
<textarea v-model="service.expected" class="form-control" rows="3" autocapitalize="none" spellcheck="false" placeholder='status:SERVING'></textarea>
<small class="form-text text-muted">Check <a target="_blank" href="https://pkg.go.dev/google.golang.org/grpc/health/grpc_health_v1?tab=doc#pkg-variables">GPRC health check response codes</a> for more information.</small>
</div>
</div>

<div v-if="service.grpc_health_check" class="form-group row">
<label for="service_response_code" class="col-sm-4 col-form-label">Expected Status Code</label>
<div class="col-sm-8">
<input v-model="service.expected_status" type="number" name="expected_status" class="form-control" placeholder="1" id="service_response_code">
<small class="form-text text-muted">A status code of 1 is success, or view all the <a target="_blank" href="https://pkg.go.dev/google.golang.org/grpc/health/grpc_health_v1?tab=doc#HealthCheckResponse_ServingStatus">GRPC Status Codes</a></small>
</div>
</div>

<div v-if="service.type.match(/^(tcp|http)$/)" class="form-group row">
<label class="col-12 col-md-4 col-form-label">{{ $t('tls_cert') }}</label>
<div class="col-12 col-md-8 mt-1 mb-2 mb-md-0">
Expand Down Expand Up @@ -276,6 +302,7 @@
permalink: "",
order: 1,
verify_ssl: true,
grpc_health_check: false,
redirect: true,
allow_notifications: true,
notify_all_changes: true,
Expand Down Expand Up @@ -316,6 +343,21 @@
this.service = this.in_service
}
this.use_tls = this.service.tls_cert !== ""
},
updateDefaultValues() {
if (this.service.type === "grpc") {
this.service.expected_status = 1
this.service.expected = "status:SERVING"
this.service.port = 50051
this.service.verify_ssl = false
this.service.method = ""
} else {
this.service.expected_status = 200
this.service.expected = ""
this.service.port = 80
this.service.verify_ssl = true
this.service.method = "GET"
}
},
updatePermalink() {
const a = 'àáâäæãåāăąçćčđďèéêëēėęěğǵḧîïíīįìłḿñńǹňôöòóœøōõőṕŕřßśšşșťțûüùúūǘůűųẃẍÿýžźż·/_,:;'
Expand Down
90 changes: 82 additions & 8 deletions types/services/routine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,25 @@ package services

import (
"bytes"
"context"
"crypto/tls"
"fmt"
"github.com/prometheus/client_golang/prometheus"
"github.com/statping/statping/types/metrics"
"google.golang.org/grpc"
"net"
"net/http"
"net/url"
"regexp"
"strings"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/statping/statping/types/metrics"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"

"github.com/statping/statping/types/failures"
"github.com/statping/statping/types/hits"
"github.com/statping/statping/utils"
healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

// checkServices will start the checking go routine for each service
Expand Down Expand Up @@ -115,13 +119,41 @@ func CheckGrpc(s *Service, record bool) (*Service, error) {
timer := prometheus.NewTimer(metrics.ServiceTimer(s.Name))
defer timer.ObserveDuration()

// Strip URL scheme if present. Eg: https:// , http://
if strings.Contains(s.Domain, "://") {
u, err := url.Parse(s.Domain)
if err != nil {
// Unable to parse.
log.Warnln(fmt.Sprintf("GRPC Service: '%s', Unable to parse URL: '%v'", s.Name, s.Domain))
if record {
RecordFailure(s, fmt.Sprintf("Unable to parse GRPC domain %v, %v", s.Domain, err), "parse_domain")
}
}

// Set domain as hostname without port number.
s.Domain = u.Hostname()
}

// Calculate DNS check time
dnsLookup, err := dnsCheck(s)
if err != nil {
if record {
RecordFailure(s, fmt.Sprintf("Could not get IP address for GRPC service %v, %v", s.Domain, err), "lookup")
}
return s, err
}

// Connect to grpc service without TLS certs.
grpcOption := grpc.WithInsecure()

// Check if TLS is enabled
// Upgrade GRPC connection if using TLS
// Force to connect on HTTP2 with TLS. Needed when using a reverse proxy such as nginx.
if s.VerifySSL.Bool {
h2creds := credentials.NewTLS(&tls.Config{NextProtos: []string{"h2"}})
grpcOption = grpc.WithTransportCredentials(h2creds)
}

s.PingTime = dnsLookup
t1 := utils.Now()
domain := fmt.Sprintf("%v", s.Domain)
Expand All @@ -131,28 +163,70 @@ func CheckGrpc(s *Service, record bool) (*Service, error) {
domain = fmt.Sprintf("[%v]:%v", s.Domain, s.Port)
}
}
conn, err := grpc.Dial(domain, grpc.WithInsecure(), grpc.WithBlock())
if err != nil {
log.Fatalf("did not connect: %v", err)
}

// Context will cancel the request when timeout is exceeded.
// Cancel the context when request is served within the timeout limit.
timeout := time.Duration(s.Timeout) * time.Second
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()

conn, err := grpc.DialContext(ctx, domain, grpcOption, grpc.WithBlock())
if err != nil {
if record {
RecordFailure(s, fmt.Sprintf("Dial Error %v", err), "connection")
}
return s, err
}

if s.GrpcHealthCheck.Bool {
// Create a new health check client
c := healthpb.NewHealthClient(conn)
in := &healthpb.HealthCheckRequest{}
res, err := c.Check(ctx, in)
if err != nil {
if record {
RecordFailure(s, fmt.Sprintf("GRPC Error %v", err), "healthcheck")
}
return s, nil
}

// Record responses
s.LastResponse = strings.TrimSpace(res.String())
s.LastStatusCode = int(res.GetStatus())
}

if err := conn.Close(); err != nil {
if record {
RecordFailure(s, fmt.Sprintf("%v Socket Close Error %v", strings.ToUpper(s.Type), err), "close")
}
return s, err
}

// Record latency
s.Latency = utils.Now().Sub(t1).Microseconds()
s.LastResponse = ""
s.Online = true

if s.GrpcHealthCheck.Bool {
if s.ExpectedStatus != s.LastStatusCode {
if record {
RecordFailure(s, fmt.Sprintf("GRPC Service: '%s', Status Code: expected '%v', got '%v'", s.Name, s.ExpectedStatus, s.LastStatusCode), "response_code")
}
return s, nil
}

if s.Expected.String != s.LastResponse {
log.Warnln(fmt.Sprintf("GRPC Service: '%s', Response: expected '%v', got '%v'", s.Name, s.Expected.String, s.LastResponse))
if record {
RecordFailure(s, fmt.Sprintf("GRPC Response Body '%v' did not match '%v'", s.LastResponse, s.Expected.String), "response_body")
}
return s, nil
}
}

if record {
RecordSuccess(s)
}

return s, nil
}

Expand Down
Loading

0 comments on commit fccc559

Please sign in to comment.