Allow config of additonal CircuitBreaker params

2024-12-22 13:34:03 +03:00 · 2022-04-05 11:30:08 +01:00 · 2022-04-05 11:30:08 +01:00 · 71150bcaaf
commit 71150bcaaf
parent 8c56d1a338
15 changed files with 218 additions and 30 deletions
--- a/docs/content/middlewares/http/circuitbreaker.md
+++ b/docs/content/middlewares/http/circuitbreaker.md
@ -171,15 +171,18 @@ This behavior cannot be configured.

 ### `CheckPeriod`

-The interval used to evaluate `expression` and decide if the state of the circuit breaker must change.
-By default, `CheckPeriod` is 100ms. This value cannot be configured.
+_Optional, Default="100ms"_
+
+The interval between successive checks of the circuit breaker condition (when in standby state).

 ### `FallbackDuration`

-By default, `FallbackDuration` is 10 seconds. This value cannot be configured.
+_Optional, Default="10s"_

-### `RecoveringDuration`
+The duration for which the circuit breaker will wait before trying to recover (from a tripped state).

-The duration of the recovering mode (recovering state).
+### `RecoveryDuration`

-By default, `RecoveringDuration` is 10 seconds. This value cannot be configured.
+_Optional, Default="10s"_
+
+The duration for which the circuit breaker will try to recover (as soon as it is in recovering state).
--- a/docs/content/reference/dynamic-configuration/docker-labels.yml
+++ b/docs/content/reference/dynamic-configuration/docker-labels.yml
@ -11,6 +11,9 @@
 - "traefik.http.middlewares.middleware02.buffering.retryexpression=foobar"
 - "traefik.http.middlewares.middleware03.chain.middlewares=foobar, foobar"
 - "traefik.http.middlewares.middleware04.circuitbreaker.expression=foobar"
+- "traefik.http.middlewares.middleware04.circuitbreaker.checkperiod=42s"
+- "traefik.http.middlewares.middleware04.circuitbreaker.fallbackduration=42s"
+- "traefik.http.middlewares.middleware04.circuitbreaker.recoveryduration=42s"
 - "traefik.http.middlewares.middleware05.compress=true"
 - "traefik.http.middlewares.middleware05.compress.excludedcontenttypes=foobar, foobar"
 - "traefik.http.middlewares.middleware05.compress.minresponsebodybytes=42"
--- a/docs/content/reference/dynamic-configuration/file.toml
+++ b/docs/content/reference/dynamic-configuration/file.toml
@ -125,6 +125,9 @@
    [http.middlewares.Middleware04]
      [http.middlewares.Middleware04.circuitBreaker]
        expression = "foobar"
+        checkPeriod = "42s"
+        fallbackDuration = "42s"
+        recoveryDuration = "42s"
    [http.middlewares.Middleware05]
      [http.middlewares.Middleware05.compress]
        excludedContentTypes = ["foobar", "foobar"]
--- a/docs/content/reference/dynamic-configuration/file.yaml
+++ b/docs/content/reference/dynamic-configuration/file.yaml
@ -128,6 +128,9 @@ http:
    Middleware04:
      circuitBreaker:
        expression: foobar
+        checkPeriod: 42s
+        fallbackDuration: 42s
+        recoveryDuration: 42s
    Middleware05:
      compress:
        excludedContentTypes:
--- a/docs/content/reference/dynamic-configuration/kv-ref.md
+++ b/docs/content/reference/dynamic-configuration/kv-ref.md
@ -12,7 +12,10 @@
 | `traefik/http/middlewares/Middleware02/buffering/retryExpression` | `foobar` |
 | `traefik/http/middlewares/Middleware03/chain/middlewares/0` | `foobar` |
 | `traefik/http/middlewares/Middleware03/chain/middlewares/1` | `foobar` |
+| `traefik/http/middlewares/Middleware04/circuitBreaker/checkPeriod` | `42s` |
 | `traefik/http/middlewares/Middleware04/circuitBreaker/expression` | `foobar` |
+| `traefik/http/middlewares/Middleware04/circuitBreaker/fallbackDuration` | `42s` |
+| `traefik/http/middlewares/Middleware04/circuitBreaker/recoveryDuration` | `42s` |
 | `traefik/http/middlewares/Middleware05/compress/excludedContentTypes/0` | `foobar` |
 | `traefik/http/middlewares/Middleware05/compress/excludedContentTypes/1` | `foobar` |
 | `traefik/http/middlewares/Middleware05/compress/minResponseBodyBytes` | `42` |
--- a/docs/content/reference/dynamic-configuration/marathon-labels.json
+++ b/docs/content/reference/dynamic-configuration/marathon-labels.json
@ -11,6 +11,9 @@
 "traefik.http.middlewares.middleware02.buffering.retryexpression": "foobar",
 "traefik.http.middlewares.middleware03.chain.middlewares": "foobar, foobar",
 "traefik.http.middlewares.middleware04.circuitbreaker.expression": "foobar",
+"traefik.http.middlewares.middleware04.circuitbreaker.checkperiod": "42s",
+"traefik.http.middlewares.middleware04.circuitbreaker.fallbackduration": "42s",
+"traefik.http.middlewares.middleware04.circuitbreaker.recoveryduration": "42s",
 "traefik.http.middlewares.middleware05.compress": "true",
 "traefik.http.middlewares.middleware05.compress.excludedcontenttypes": "foobar, foobar",
 "traefik.http.middlewares.middleware05.compress.minresponsebodybytes": "42",
--- a/docs/content/reference/dynamic-configuration/traefik.containo.us_middlewares.yaml
+++ b/docs/content/reference/dynamic-configuration/traefik.containo.us_middlewares.yaml
@ -91,8 +91,32 @@ spec:
              circuitBreaker:
                description: CircuitBreaker holds the circuit breaker configuration.
                properties:
+                  checkPeriod:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    description: CheckPeriod is the interval between successive checks
+                      of the circuit breaker condition (when in standby state).
+                    x-kubernetes-int-or-string: true
                  expression:
+                    description: Expression is the condition that triggers the tripped
+                      state.
                    type: string
+                  fallbackDuration:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    description: FallbackDuration is the duration for which the circuit
+                      breaker will wait before trying to recover (from a tripped state).
+                    x-kubernetes-int-or-string: true
+                  recoveryDuration:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    description: RecoveryDuration is the duration for which the circuit
+                      breaker will try to recover (as soon as it is in recovering
+                      state).
+                    x-kubernetes-int-or-string: true
                type: object
              compress:
                description: Compress holds the compress configuration.
--- a/integration/fixtures/k8s/01-traefik-crd.yml
+++ b/integration/fixtures/k8s/01-traefik-crd.yml
@ -535,8 +535,32 @@ spec:
              circuitBreaker:
                description: CircuitBreaker holds the circuit breaker configuration.
                properties:
+                  checkPeriod:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    description: CheckPeriod is the interval between successive checks
+                      of the circuit breaker condition (when in standby state).
+                    x-kubernetes-int-or-string: true
                  expression:
+                    description: Expression is the condition that triggers the tripped
+                      state.
                    type: string
+                  fallbackDuration:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    description: FallbackDuration is the duration for which the circuit
+                      breaker will wait before trying to recover (from a tripped state).
+                    x-kubernetes-int-or-string: true
+                  recoveryDuration:
+                    anyOf:
+                    - type: integer
+                    - type: string
+                    description: RecoveryDuration is the duration for which the circuit
+                      breaker will try to recover (as soon as it is in recovering
+                      state).
+                    x-kubernetes-int-or-string: true
                type: object
              compress:
                description: Compress holds the compress configuration.
--- a/pkg/config/dynamic/middlewares.go
+++ b/pkg/config/dynamic/middlewares.go
@ -93,7 +93,21 @@ type Chain struct {

 // CircuitBreaker holds the circuit breaker configuration.
 type CircuitBreaker struct {
+	// Expression is the condition that triggers the tripped state.
 	Expression string `json:"expression,omitempty" toml:"expression,omitempty" yaml:"expression,omitempty" export:"true"`
+	// CheckPeriod is the interval between successive checks of the circuit breaker condition (when in standby state).
+	CheckPeriod ptypes.Duration `json:"checkPeriod,omitempty" toml:"checkPeriod,omitempty" yaml:"checkPeriod,omitempty" export:"true"`
+	// FallbackDuration is the duration for which the circuit breaker will wait before trying to recover (from a tripped state).
+	FallbackDuration ptypes.Duration `json:"fallbackDuration,omitempty" toml:"fallbackDuration,omitempty" yaml:"fallbackDuration,omitempty" export:"true"`
+	// RecoveryDuration is the duration for which the circuit breaker will try to recover (as soon as it is in recovering state).
+	RecoveryDuration ptypes.Duration `json:"recoveryDuration,omitempty" toml:"recoveryDuration,omitempty" yaml:"recoveryDuration,omitempty" export:"true"`
+}
+
+// SetDefaults sets the default values on a RateLimit.
+func (c *CircuitBreaker) SetDefaults() {
+	c.CheckPeriod = ptypes.Duration(100 * time.Millisecond)
+	c.FallbackDuration = ptypes.Duration(10 * time.Second)
+	c.RecoveryDuration = ptypes.Duration(10 * time.Second)
 }

 // +k8s:deepcopy-gen=true
--- a/pkg/config/label/label_test.go
+++ b/pkg/config/label/label_test.go
@ -27,6 +27,9 @@ func TestDecodeConfiguration(t *testing.T) {
 		"traefik.http.middlewares.Middleware2.buffering.retryexpression":                           "foobar",
 		"traefik.http.middlewares.Middleware3.chain.middlewares":                                   "foobar, fiibar",
 		"traefik.http.middlewares.Middleware4.circuitbreaker.expression":                           "foobar",
+		"traefik.HTTP.Middlewares.Middleware4.circuitbreaker.checkperiod":                          "1s",
+		"traefik.HTTP.Middlewares.Middleware4.circuitbreaker.fallbackduration":                     "1s",
+		"traefik.HTTP.Middlewares.Middleware4.circuitbreaker.recoveryduration":                     "1s",
 		"traefik.http.middlewares.Middleware5.digestauth.headerfield":                              "foobar",
 		"traefik.http.middlewares.Middleware5.digestauth.realm":                                    "foobar",
 		"traefik.http.middlewares.Middleware5.digestauth.removeheader":                             "true",
@ -488,7 +491,10 @@ func TestDecodeConfiguration(t *testing.T) {
 				},
 				"Middleware4": {
 					CircuitBreaker: &dynamic.CircuitBreaker{
-						Expression: "foobar",
+						Expression:       "foobar",
+						CheckPeriod:      ptypes.Duration(time.Second),
+						FallbackDuration: ptypes.Duration(time.Second),
+						RecoveryDuration: ptypes.Duration(time.Second),
 					},
 				},
 				"Middleware5": {
@ -983,7 +989,10 @@ func TestEncodeConfiguration(t *testing.T) {
 				},
 				"Middleware4": {
 					CircuitBreaker: &dynamic.CircuitBreaker{
-						Expression: "foobar",
+						Expression:       "foobar",
+						CheckPeriod:      ptypes.Duration(time.Second),
+						FallbackDuration: ptypes.Duration(time.Second),
+						RecoveryDuration: ptypes.Duration(time.Second),
 					},
 				},
 				"Middleware5": {
@ -1191,6 +1200,9 @@ func TestEncodeConfiguration(t *testing.T) {
 		"traefik.HTTP.Middlewares.Middleware2.Buffering.RetryExpression":                           "foobar",
 		"traefik.HTTP.Middlewares.Middleware3.Chain.Middlewares":                                   "foobar, fiibar",
 		"traefik.HTTP.Middlewares.Middleware4.CircuitBreaker.Expression":                           "foobar",
+		"traefik.HTTP.Middlewares.Middleware4.CircuitBreaker.CheckPeriod":                          "1000000000",
+		"traefik.HTTP.Middlewares.Middleware4.CircuitBreaker.FallbackDuration":                     "1000000000",
+		"traefik.HTTP.Middlewares.Middleware4.CircuitBreaker.RecoveryDuration":                     "1000000000",
 		"traefik.HTTP.Middlewares.Middleware5.DigestAuth.HeaderField":                              "foobar",
 		"traefik.HTTP.Middlewares.Middleware5.DigestAuth.Realm":                                    "foobar",
 		"traefik.HTTP.Middlewares.Middleware5.DigestAuth.RemoveHeader":                             "true",
--- a/pkg/middlewares/circuitbreaker/circuit_breaker.go
+++ b/pkg/middlewares/circuitbreaker/circuit_breaker.go
@ -3,6 +3,7 @@ package circuitbreaker
 import (
 	"context"
 	"net/http"
+	"time"

 	"github.com/opentracing/opentracing-go/ext"
 	"github.com/traefik/traefik/v2/pkg/config/dynamic"
@ -12,9 +13,7 @@ import (
 	"github.com/vulcand/oxy/cbreaker"
 )

-const (
-	typeName = "CircuitBreaker"
-)
+const typeName = "CircuitBreaker"

 type circuitBreaker struct {
 	circuitBreaker *cbreaker.CircuitBreaker
@ -27,9 +26,32 @@ func New(ctx context.Context, next http.Handler, confCircuitBreaker dynamic.Circ

 	logger := log.FromContext(middlewares.GetLoggerCtx(ctx, name, typeName))
 	logger.Debug("Creating middleware")
-	logger.Debug("Setting up with expression: %s", expression)
+	logger.Debugf("Setting up with expression: %s", expression)

-	oxyCircuitBreaker, err := cbreaker.New(next, expression, createCircuitBreakerOptions(expression))
+	cbOpts := []cbreaker.CircuitBreakerOption{
+		cbreaker.Fallback(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+			tracing.SetErrorWithEvent(req, "blocked by circuit-breaker (%q)", expression)
+			rw.WriteHeader(http.StatusServiceUnavailable)
+
+			if _, err := rw.Write([]byte(http.StatusText(http.StatusServiceUnavailable))); err != nil {
+				log.FromContext(req.Context()).Error(err)
+			}
+		})),
+	}
+
+	if confCircuitBreaker.CheckPeriod > 0 {
+		cbOpts = append(cbOpts, cbreaker.CheckPeriod(time.Duration(confCircuitBreaker.CheckPeriod)))
+	}
+
+	if confCircuitBreaker.FallbackDuration > 0 {
+		cbOpts = append(cbOpts, cbreaker.FallbackDuration(time.Duration(confCircuitBreaker.FallbackDuration)))
+	}
+
+	if confCircuitBreaker.RecoveryDuration > 0 {
+		cbOpts = append(cbOpts, cbreaker.RecoveryDuration(time.Duration(confCircuitBreaker.RecoveryDuration)))
+	}
+
+	oxyCircuitBreaker, err := cbreaker.New(next, expression, cbOpts...)
 	if err != nil {
 		return nil, err
 	}
@ -39,18 +61,6 @@ func New(ctx context.Context, next http.Handler, confCircuitBreaker dynamic.Circ
 	}, nil
 }

-// NewCircuitBreakerOptions returns a new CircuitBreakerOption.
-func createCircuitBreakerOptions(expression string) cbreaker.CircuitBreakerOption {
-	return cbreaker.Fallback(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
-		tracing.SetErrorWithEvent(req, "blocked by circuit-breaker (%q)", expression)
-		rw.WriteHeader(http.StatusServiceUnavailable)
-
-		if _, err := rw.Write([]byte(http.StatusText(http.StatusServiceUnavailable))); err != nil {
-			log.FromContext(req.Context()).Error(err)
-		}
-	}))
-}
-
 func (c *circuitBreaker) GetTracingInformation() (string, ext.SpanKindEnum) {
 	return c.name, tracing.SpanKindNoneEnum
 }
--- a/pkg/provider/kubernetes/crd/kubernetes.go
+++ b/pkg/provider/kubernetes/crd/kubernetes.go
@ -243,6 +243,12 @@ func (p *Provider) loadConfigurationFromCRD(ctx context.Context, client Client)
 			continue
 		}

+		circuitBreaker, err := createCircuitBreakerMiddleware(middleware.Spec.CircuitBreaker)
+		if err != nil {
+			log.FromContext(ctxMid).Errorf("Error while reading circuit breaker middleware: %v", err)
+			continue
+		}
+
 		conf.HTTP.Middlewares[id] = &dynamic.Middleware{
 			AddPrefix:         middleware.Spec.AddPrefix,
 			StripPrefix:       middleware.Spec.StripPrefix,
@ -261,7 +267,7 @@ func (p *Provider) loadConfigurationFromCRD(ctx context.Context, client Client)
 			ForwardAuth:       forwardAuth,
 			InFlightReq:       middleware.Spec.InFlightReq,
 			Buffering:         middleware.Spec.Buffering,
-			CircuitBreaker:    middleware.Spec.CircuitBreaker,
+			CircuitBreaker:    circuitBreaker,
 			Compress:          middleware.Spec.Compress,
 			PassTLSClientCert: middleware.Spec.PassTLSClientCert,
 			Retry:             retry,
@ -425,6 +431,35 @@ func createPluginMiddleware(plugins map[string]apiextensionv1.JSON) (map[string]
 	return pc, nil
 }

+func createCircuitBreakerMiddleware(circuitBreaker *v1alpha1.CircuitBreaker) (*dynamic.CircuitBreaker, error) {
+	if circuitBreaker == nil {
+		return nil, nil
+	}
+
+	cb := &dynamic.CircuitBreaker{Expression: circuitBreaker.Expression}
+	cb.SetDefaults()
+
+	if circuitBreaker.CheckPeriod != nil {
+		if err := cb.CheckPeriod.Set(circuitBreaker.CheckPeriod.String()); err != nil {
+			return nil, err
+		}
+	}
+
+	if circuitBreaker.FallbackDuration != nil {
+		if err := cb.FallbackDuration.Set(circuitBreaker.FallbackDuration.String()); err != nil {
+			return nil, err
+		}
+	}
+
+	if circuitBreaker.RecoveryDuration != nil {
+		if err := cb.RecoveryDuration.Set(circuitBreaker.RecoveryDuration.String()); err != nil {
+			return nil, err
+		}
+	}
+
+	return cb, nil
+}
+
 func createRateLimitMiddleware(rateLimit *v1alpha1.RateLimit) (*dynamic.RateLimit, error) {
 	if rateLimit == nil {
 		return nil, nil
--- a/pkg/provider/kubernetes/crd/traefik/v1alpha1/middleware.go
+++ b/pkg/provider/kubernetes/crd/traefik/v1alpha1/middleware.go
@ -40,7 +40,7 @@ type MiddlewareSpec struct {
 	ForwardAuth       *ForwardAuth                   `json:"forwardAuth,omitempty"`
 	InFlightReq       *dynamic.InFlightReq           `json:"inFlightReq,omitempty"`
 	Buffering         *dynamic.Buffering             `json:"buffering,omitempty"`
-	CircuitBreaker    *dynamic.CircuitBreaker        `json:"circuitBreaker,omitempty"`
+	CircuitBreaker    *CircuitBreaker                `json:"circuitBreaker,omitempty"`
 	Compress          *dynamic.Compress              `json:"compress,omitempty"`
 	PassTLSClientCert *dynamic.PassTLSClientCert     `json:"passTLSClientCert,omitempty"`
 	Retry             *Retry                         `json:"retry,omitempty"`
@ -59,6 +59,20 @@ type ErrorPage struct {

 // +k8s:deepcopy-gen=true

+// CircuitBreaker holds the circuit breaker configuration.
+type CircuitBreaker struct {
+	// Expression is the condition that triggers the tripped state.
+	Expression string `json:"expression,omitempty" toml:"expression,omitempty" yaml:"expression,omitempty" export:"true"`
+	// CheckPeriod is the interval between successive checks of the circuit breaker condition (when in standby state).
+	CheckPeriod *intstr.IntOrString `json:"checkPeriod,omitempty" toml:"checkPeriod,omitempty" yaml:"checkPeriod,omitempty" export:"true"`
+	// FallbackDuration is the duration for which the circuit breaker will wait before trying to recover (from a tripped state).
+	FallbackDuration *intstr.IntOrString `json:"fallbackDuration,omitempty" toml:"fallbackDuration,omitempty" yaml:"fallbackDuration,omitempty" export:"true"`
+	// RecoveryDuration is the duration for which the circuit breaker will try to recover (as soon as it is in recovering state).
+	RecoveryDuration *intstr.IntOrString `json:"recoveryDuration,omitempty" toml:"recoveryDuration,omitempty" yaml:"recoveryDuration,omitempty" export:"true"`
+}
+
+// +k8s:deepcopy-gen=true
+
 // Chain holds a chain of middlewares.
 type Chain struct {
 	Middlewares []MiddlewareRef `json:"middlewares,omitempty"`
--- a/pkg/provider/kubernetes/crd/traefik/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/provider/kubernetes/crd/traefik/v1alpha1/zz_generated.deepcopy.go
@ -74,6 +74,37 @@ func (in *Chain) DeepCopy() *Chain {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CircuitBreaker) DeepCopyInto(out *CircuitBreaker) {
+	*out = *in
+	if in.CheckPeriod != nil {
+		in, out := &in.CheckPeriod, &out.CheckPeriod
+		*out = new(intstr.IntOrString)
+		**out = **in
+	}
+	if in.FallbackDuration != nil {
+		in, out := &in.FallbackDuration, &out.FallbackDuration
+		*out = new(intstr.IntOrString)
+		**out = **in
+	}
+	if in.RecoveryDuration != nil {
+		in, out := &in.RecoveryDuration, &out.RecoveryDuration
+		*out = new(intstr.IntOrString)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CircuitBreaker.
+func (in *CircuitBreaker) DeepCopy() *CircuitBreaker {
+	if in == nil {
+		return nil
+	}
+	out := new(CircuitBreaker)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ClientAuth) DeepCopyInto(out *ClientAuth) {
 	*out = *in
@ -714,8 +745,8 @@ func (in *MiddlewareSpec) DeepCopyInto(out *MiddlewareSpec) {
 	}
 	if in.CircuitBreaker != nil {
 		in, out := &in.CircuitBreaker, &out.CircuitBreaker
-		*out = new(dynamic.CircuitBreaker)
-		**out = **in
+		*out = new(CircuitBreaker)
+		(*in).DeepCopyInto(*out)
 	}
 	if in.Compress != nil {
 		in, out := &in.Compress, &out.Compress
--- a/pkg/provider/kv/kv_test.go
+++ b/pkg/provider/kv/kv_test.go
@ -173,6 +173,9 @@ func Test_buildConfiguration(t *testing.T) {
 		"traefik/http/middlewares/Middleware03/chain/middlewares/0":                                  "foobar",
 		"traefik/http/middlewares/Middleware03/chain/middlewares/1":                                  "foobar",
 		"traefik/http/middlewares/Middleware04/circuitBreaker/expression":                            "foobar",
+		"traefik/http/middlewares/Middleware04/circuitBreaker/checkPeriod":                           "1s",
+		"traefik/http/middlewares/Middleware04/circuitBreaker/fallbackDuration":                      "1s",
+		"traefik/http/middlewares/Middleware04/circuitBreaker/recoveryDuration":                      "1s",
 		"traefik/http/middlewares/Middleware07/errors/status/0":                                      "foobar",
 		"traefik/http/middlewares/Middleware07/errors/status/1":                                      "foobar",
 		"traefik/http/middlewares/Middleware07/errors/service":                                       "foobar",
@ -393,7 +396,10 @@ func Test_buildConfiguration(t *testing.T) {
 				},
 				"Middleware04": {
 					CircuitBreaker: &dynamic.CircuitBreaker{
-						Expression: "foobar",
+						Expression:       "foobar",
+						CheckPeriod:      ptypes.Duration(time.Second),
+						FallbackDuration: ptypes.Duration(time.Second),
+						RecoveryDuration: ptypes.Duration(time.Second),
 					},
 				},
 				"Middleware05": {