shaba/traefik
1
0
Fork 0
mirror of https://github.com/containous/traefik.git synced 2024-12-23 17:34:13 +03:00
Code Releases Activity

Re-add server up metrics

Browse Source
This commit is contained in:
Robin Müller 2020-09-26 13:30:03 +02:00 committed by GitHub
parent 5e63ab619e
commit f2e53a3569
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 36 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
pkg/healthcheck/healthcheck.go
View File

@ -10,9 +10,10 @@ import (
"sync"
"time"
"github.com/go-kit/kit/metrics"
gokitmetrics "github.com/go-kit/kit/metrics"
"github.com/traefik/traefik/v2/pkg/config/runtime"
"github.com/traefik/traefik/v2/pkg/log"
"github.com/traefik/traefik/v2/pkg/metrics"
"github.com/traefik/traefik/v2/pkg/safe"
"github.com/vulcand/oxy/roundrobin"
)
@ -40,11 +41,8 @@ type BalancerHandler interface {
Balancer
}
// metricsRegistry is a local interface in the health check package,
// exposing only the required metrics necessary for the health check package.
// This makes it easier for the tests.
type metricsRegistry interface {
BackendServerUpGauge() metrics.Gauge
type metricsHealthcheck struct {
serverUpGauge gokitmetrics.Gauge
}
// Options are the public health check options.
@ -109,7 +107,7 @@ func (b *BackendConfig) addHeadersAndHost(req *http.Request) *http.Request {
// HealthCheck struct.
type HealthCheck struct {
Backends map[string]*BackendConfig
metrics metricsRegistry
metrics metricsHealthcheck
cancel context.CancelFunc
}
@ -153,22 +151,33 @@ func (hc *HealthCheck) checkBackend(ctx context.Context, backend *BackendConfig)
logger := log.FromContext(ctx)
enabledURLs := backend.LB.Servers()
var newDisabledURLs []backendURL
for _, disabledURL := range backend.disabledURLs {
serverUpMetricValue := float64(0)
if err := checkHealth(disabledURL.url, backend); err == nil {
logger.Warnf("Health check up: Returning to server list. Backend: %q URL: %q Weight: %d",
backend.name, disabledURL.url.String(), disabledURL.weight)
if err = backend.LB.UpsertServer(disabledURL.url, roundrobin.Weight(disabledURL.weight)); err != nil {
logger.Error(err)
}
serverUpMetricValue = 1
} else {
logger.Warnf("Health check still failing. Backend: %q URL: %q Reason: %s", backend.name, disabledURL.url.String(), err)
newDisabledURLs = append(newDisabledURLs, disabledURL)
}
labelValues := []string{"service", backend.name, "url", disabledURL.url.String()}
hc.metrics.serverUpGauge.With(labelValues...).Set(serverUpMetricValue)
}
backend.disabledURLs = newDisabledURLs
for _, enableURL := range enabledURLs {
serverUpMetricValue := float64(1)
if err := checkHealth(enableURL, backend); err != nil {
weight := 1
rr, ok := backend.LB.(*roundrobin.RoundRobin)
@ -179,26 +188,36 @@ func (hc *HealthCheck) checkBackend(ctx context.Context, backend *BackendConfig)
weight = 1
}
}
logger.Warnf("Health check failed, removing from server list. Backend: %q URL: %q Weight: %d Reason: %s", backend.name, enableURL.String(), weight, err)
logger.Warnf("Health check failed, removing from server list. Backend: %q URL: %q Weight: %d Reason: %s",
backend.name, enableURL.String(), weight, err)
if err := backend.LB.RemoveServer(enableURL); err != nil {
logger.Error(err)
}
backend.disabledURLs = append(backend.disabledURLs, backendURL{enableURL, weight})
serverUpMetricValue = 0
}
labelValues := []string{"service", backend.name, "url", enableURL.String()}
hc.metrics.serverUpGauge.With(labelValues...).Set(serverUpMetricValue)
}
}
// GetHealthCheck returns the health check which is guaranteed to be a singleton.
func GetHealthCheck() *HealthCheck {
func GetHealthCheck(registry metrics.Registry) *HealthCheck {
once.Do(func() {
singleton = newHealthCheck()
singleton = newHealthCheck(registry)
})
return singleton
}
func newHealthCheck() *HealthCheck {
func newHealthCheck(registry metrics.Registry) *HealthCheck {
return &HealthCheck{
Backends: make(map[string]*BackendConfig),
metrics: metricsHealthcheck{
serverUpGauge: registry.ServiceServerUpGauge(),
},
}
}

10
pkg/healthcheck/healthcheck_test.go
View File

@ -120,10 +120,10 @@ func TestSetBackendsConfiguration(t *testing.T) {
backend.disabledURLs = append(backend.disabledURLs, backendURL{url: serverURL, weight: 1})
}
collectingMetrics := testhelpers.NewCollectingHealthCheckMetrics()
collectingMetrics := &testhelpers.CollectingGauge{}
check := HealthCheck{
Backends: make(map[string]*BackendConfig),
metrics: collectingMetrics,
metrics: metricsHealthcheck{serverUpGauge: collectingMetrics},
}
wg := sync.WaitGroup{}
@ -149,8 +149,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
assert.Equal(t, test.expectedNumRemovedServers, lb.numRemovedServers, "removed servers")
assert.Equal(t, test.expectedNumUpsertedServers, lb.numUpsertedServers, "upserted servers")
// FIXME re add metrics
// assert.Equal(t, test.expectedGaugeValue, collectingMetrics.Gauge.GaugeValue, "ServerUp Gauge")
assert.Equal(t, test.expectedGaugeValue, collectingMetrics.GaugeValue, "ServerUp Gauge")
})
}
}
@ -502,9 +501,10 @@ func TestNotFollowingRedirects(t *testing.T) {
FollowRedirects: false,
}, "backendName")
collectingMetrics := &testhelpers.CollectingGauge{}
check := HealthCheck{
Backends: make(map[string]*BackendConfig),
metrics: testhelpers.NewCollectingHealthCheckMetrics(),
metrics: metricsHealthcheck{serverUpGauge: collectingMetrics},
}
wg := sync.WaitGroup{}

3
pkg/server/service/service.go
View File

@ -236,8 +236,7 @@ func (m *Manager) LaunchHealthCheck() {
}
}
// FIXME metrics and context
healthcheck.GetHealthCheck().SetBackendsConfiguration(context.Background(), backendConfigs)
healthcheck.GetHealthCheck(m.metricsRegistry).SetBackendsConfiguration(context.Background(), backendConfigs)
}
func buildHealthCheckOptions(ctx context.Context, lb healthcheck.Balancer, backend string, hc *dynamic.HealthCheck) *healthcheck.Options {