From b45a6b5f13c2834c39f3850a3ad88c208d82970d Mon Sep 17 00:00:00 2001 From: Leonardo Helman Date: Tue, 9 Feb 2021 19:33:16 -0300 Subject: [PATCH] Adding circuit breaker (#64) * adding configurable http client timeout * added circuit breaker * added gobreaker license * vendored gobreaker * addressed all the pr suggestions * moved the circuit braker from api to processor * added a test for the circuit breaker opening * changed consecutive with total failures to trigger --- LICENSE-3rdparty.csv | 1 + ddlambda.go | 16 + go.mod | 1 + go.sum | 1 + internal/metrics/api.go | 14 +- internal/metrics/constants.go | 12 +- internal/metrics/listener.go | 41 ++- internal/metrics/processor.go | 49 ++- internal/metrics/processor_test.go | 35 +- vendor/github.com/sony/gobreaker/.travis.yml | 15 + vendor/github.com/sony/gobreaker/LICENSE | 21 ++ vendor/github.com/sony/gobreaker/README.md | 128 +++++++ vendor/github.com/sony/gobreaker/go.mod | 5 + vendor/github.com/sony/gobreaker/go.sum | 7 + vendor/github.com/sony/gobreaker/gobreaker.go | 344 ++++++++++++++++++ 15 files changed, 652 insertions(+), 38 deletions(-) create mode 100644 vendor/github.com/sony/gobreaker/.travis.yml create mode 100644 vendor/github.com/sony/gobreaker/LICENSE create mode 100644 vendor/github.com/sony/gobreaker/README.md create mode 100644 vendor/github.com/sony/gobreaker/go.mod create mode 100644 vendor/github.com/sony/gobreaker/go.sum create mode 100644 vendor/github.com/sony/gobreaker/gobreaker.go diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 95a5b26e..80a2c2ce 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -8,3 +8,4 @@ go-spew,github.com/davecgh/go-spew,ISC,"Copyright (c) 2012-2016 Dave Collins circuitBreakerTotalFailures } + + st := gobreaker.Settings{ + Name: "post distribution_points", + Interval: circuitBreakerInterval, + Timeout: circuitBreakerTimeout, + ReadyToTrip: readyToTrip, + } + return gobreaker.NewCircuitBreaker(st) } func (p *processor) AddMetric(metric Metric) { @@ -125,18 +144,24 @@ func (p *processor) processMetrics() { } if shouldSendBatch { - if shouldExit && p.shouldRetryOnFail { - // If we are shutting down, and we just failed to send our last batch, do a retry - bo := backoff.WithMaxRetries(backoff.NewConstantBackOff(defaultRetryInterval), 2) - err := backoff.Retry(p.sendMetricsBatch, bo) - if err != nil { - logger.Error(fmt.Errorf("failed to flush metrics to datadog API after retry: %v", err)) - } - } else { - err := p.sendMetricsBatch() - if err != nil { - logger.Error(fmt.Errorf("failed to flush metrics to datadog API: %v", err)) + _, err := p.breaker.Execute(func() (interface{}, error) { + if shouldExit && p.shouldRetryOnFail { + // If we are shutting down, and we just failed to send our last batch, do a retry + bo := backoff.WithMaxRetries(backoff.NewConstantBackOff(defaultRetryInterval), 2) + err := backoff.Retry(p.sendMetricsBatch, bo) + if err != nil { + return nil, fmt.Errorf("after retry: %v", err) + } + } else { + err := p.sendMetricsBatch() + if err != nil { + return nil, fmt.Errorf("with no retry: %v", err) + } } + return nil, nil + }) + if err != nil { + logger.Error(fmt.Errorf("failed to flush metrics to datadog API: %v", err)) } } } diff --git a/internal/metrics/processor_test.go b/internal/metrics/processor_test.go index 9ffcc497..462d57e8 100644 --- a/internal/metrics/processor_test.go +++ b/internal/metrics/processor_test.go @@ -11,6 +11,7 @@ package metrics import ( "context" "errors" + "math" "testing" "time" @@ -67,7 +68,7 @@ func TestProcessorBatches(t *testing.T) { mts.now, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") nowUnix := float64(mts.now.Unix()) - processor := MakeProcessor(context.Background(), &mc, &mts, 1000, false) + processor := MakeProcessor(context.Background(), &mc, &mts, 1000, false, time.Hour*1000, time.Hour*1000, math.MaxUint32) d1 := Distribution{ Name: "metric-1", @@ -113,7 +114,7 @@ func TestProcessorBatchesPerTick(t *testing.T) { secondTimeUnix := float64(secondTime.Unix()) mts.now = firstTime - processor := MakeProcessor(context.Background(), &mc, &mts, 1000, false) + processor := MakeProcessor(context.Background(), &mc, &mts, 1000, false, time.Hour*1000, time.Hour*1000, math.MaxUint32) d1 := Distribution{ Name: "metric-1", @@ -188,7 +189,7 @@ func TestProcessorPerformsRetry(t *testing.T) { mts.now, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") shouldRetry := true - processor := MakeProcessor(context.Background(), &mc, &mts, 1000, shouldRetry) + processor := MakeProcessor(context.Background(), &mc, &mts, 1000, shouldRetry, time.Hour*1000, time.Hour*1000, math.MaxUint32) d1 := Distribution{ Name: "metric-1", @@ -213,7 +214,7 @@ func TestProcessorCancelsWithContext(t *testing.T) { shouldRetry := true ctx, cancelFunc := context.WithCancel(context.Background()) - processor := MakeProcessor(ctx, &mc, &mts, 1000, shouldRetry) + processor := MakeProcessor(ctx, &mc, &mts, 1000, shouldRetry, time.Hour*1000, time.Hour*1000, math.MaxUint32) d1 := Distribution{ Name: "metric-1", @@ -230,3 +231,29 @@ func TestProcessorCancelsWithContext(t *testing.T) { assert.Equal(t, 0, mc.sendMetricsCalledCount) } + +func TestProcessorBatchesWithOpeningCircuitBreaker(t *testing.T) { + mc := makeMockClient() + mts := makeMockTimeService() + + mts.now, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z") + + // Will open the circuit breaker at number of total failures > 1 + circuitBreakerTotalFailures := uint32(1) + processor := MakeProcessor(context.Background(), &mc, &mts, 1000, false, time.Hour*1000, time.Hour*1000, circuitBreakerTotalFailures) + + d1 := Distribution{ + Name: "metric-1", + Tags: []string{"a", "b", "c"}, + Values: []MetricValue{{Timestamp: mts.now, Value: 1}, {Timestamp: mts.now, Value: 2}, {Timestamp: mts.now, Value: 3}}, + } + + mc.err = errors.New("Some error") + + processor.AddMetric(&d1) + + processor.FinishProcessing() + + // It should have retried 3 times, but circuit breaker opened at the second time + assert.Equal(t, 1, mc.sendMetricsCalledCount) +} diff --git a/vendor/github.com/sony/gobreaker/.travis.yml b/vendor/github.com/sony/gobreaker/.travis.yml new file mode 100644 index 00000000..6418d181 --- /dev/null +++ b/vendor/github.com/sony/gobreaker/.travis.yml @@ -0,0 +1,15 @@ +language: go +go: + - 1.10.x + - 1.11.x + - 1.12.x +sudo: false +before_install: + - go get -u golang.org/x/lint/golint + - go get github.com/axw/gocov/gocov + - go get github.com/mattn/goveralls +script: + - test -z "`gofmt -l .`" + - test -z "`golint ./...`" + - $GOPATH/bin/goveralls -service=travis-ci + - cd example && go build -o http_breaker && ./http_breaker diff --git a/vendor/github.com/sony/gobreaker/LICENSE b/vendor/github.com/sony/gobreaker/LICENSE new file mode 100644 index 00000000..81795bf6 --- /dev/null +++ b/vendor/github.com/sony/gobreaker/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright 2015 Sony Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/sony/gobreaker/README.md b/vendor/github.com/sony/gobreaker/README.md new file mode 100644 index 00000000..ce7d7a77 --- /dev/null +++ b/vendor/github.com/sony/gobreaker/README.md @@ -0,0 +1,128 @@ +gobreaker +========= + +[![GoDoc](https://godoc.org/github.com/sony/gobreaker?status.svg)](http://godoc.org/github.com/sony/gobreaker) +[![Build Status](https://travis-ci.org/sony/gobreaker.svg?branch=master)](https://travis-ci.org/sony/gobreaker) +[![Coverage Status](https://coveralls.io/repos/sony/gobreaker/badge.svg?branch=master&service=github)](https://coveralls.io/github/sony/gobreaker?branch=master) + +[gobreaker][repo-url] implements the [Circuit Breaker pattern](https://msdn.microsoft.com/en-us/library/dn589784.aspx) in Go. + +Installation +------------ + +``` +go get github.com/sony/gobreaker +``` + +Usage +----- + +The struct `CircuitBreaker` is a state machine to prevent sending requests that are likely to fail. +The function `NewCircuitBreaker` creates a new `CircuitBreaker`. + +```go +func NewCircuitBreaker(st Settings) *CircuitBreaker +``` + +You can configure `CircuitBreaker` by the struct `Settings`: + +```go +type Settings struct { + Name string + MaxRequests uint32 + Interval time.Duration + Timeout time.Duration + ReadyToTrip func(counts Counts) bool + OnStateChange func(name string, from State, to State) +} +``` + +- `Name` is the name of the `CircuitBreaker`. + +- `MaxRequests` is the maximum number of requests allowed to pass through + when the `CircuitBreaker` is half-open. + If `MaxRequests` is 0, `CircuitBreaker` allows only 1 request. + +- `Interval` is the cyclic period of the closed state + for `CircuitBreaker` to clear the internal `Counts`, described later in this section. + If `Interval` is 0, `CircuitBreaker` doesn't clear the internal `Counts` during the closed state. + +- `Timeout` is the period of the open state, + after which the state of `CircuitBreaker` becomes half-open. + If `Timeout` is 0, the timeout value of `CircuitBreaker` is set to 60 seconds. + +- `ReadyToTrip` is called with a copy of `Counts` whenever a request fails in the closed state. + If `ReadyToTrip` returns true, `CircuitBreaker` will be placed into the open state. + If `ReadyToTrip` is `nil`, default `ReadyToTrip` is used. + Default `ReadyToTrip` returns true when the number of consecutive failures is more than 5. + +- `OnStateChange` is called whenever the state of `CircuitBreaker` changes. + +The struct `Counts` holds the numbers of requests and their successes/failures: + +```go +type Counts struct { + Requests uint32 + TotalSuccesses uint32 + TotalFailures uint32 + ConsecutiveSuccesses uint32 + ConsecutiveFailures uint32 +} +``` + +`CircuitBreaker` clears the internal `Counts` either +on the change of the state or at the closed-state intervals. +`Counts` ignores the results of the requests sent before clearing. + +`CircuitBreaker` can wrap any function to send a request: + +```go +func (cb *CircuitBreaker) Execute(req func() (interface{}, error)) (interface{}, error) +``` + +The method `Execute` runs the given request if `CircuitBreaker` accepts it. +`Execute` returns an error instantly if `CircuitBreaker` rejects the request. +Otherwise, `Execute` returns the result of the request. +If a panic occurs in the request, `CircuitBreaker` handles it as an error +and causes the same panic again. + +Example +------- + +```go +var cb *breaker.CircuitBreaker + +func Get(url string) ([]byte, error) { + body, err := cb.Execute(func() (interface{}, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return body, nil + }) + if err != nil { + return nil, err + } + + return body.([]byte), nil +} +``` + +See [example](https://github.com/sony/gobreaker/blob/master/example) for details. + +License +------- + +The MIT License (MIT) + +See [LICENSE](https://github.com/sony/gobreaker/blob/master/LICENSE) for details. + + +[repo-url]: https://github.com/sony/gobreaker diff --git a/vendor/github.com/sony/gobreaker/go.mod b/vendor/github.com/sony/gobreaker/go.mod new file mode 100644 index 00000000..4367bedb --- /dev/null +++ b/vendor/github.com/sony/gobreaker/go.mod @@ -0,0 +1,5 @@ +module github.com/sony/gobreaker + +go 1.12 + +require github.com/stretchr/testify v1.3.0 diff --git a/vendor/github.com/sony/gobreaker/go.sum b/vendor/github.com/sony/gobreaker/go.sum new file mode 100644 index 00000000..4347755a --- /dev/null +++ b/vendor/github.com/sony/gobreaker/go.sum @@ -0,0 +1,7 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= diff --git a/vendor/github.com/sony/gobreaker/gobreaker.go b/vendor/github.com/sony/gobreaker/gobreaker.go new file mode 100644 index 00000000..faea6588 --- /dev/null +++ b/vendor/github.com/sony/gobreaker/gobreaker.go @@ -0,0 +1,344 @@ +// Package gobreaker implements the Circuit Breaker pattern. +// See https://msdn.microsoft.com/en-us/library/dn589784.aspx. +package gobreaker + +import ( + "errors" + "fmt" + "sync" + "time" +) + +// State is a type that represents a state of CircuitBreaker. +type State int + +// These constants are states of CircuitBreaker. +const ( + StateClosed State = iota + StateHalfOpen + StateOpen +) + +var ( + // ErrTooManyRequests is returned when the CB state is half open and the requests count is over the cb maxRequests + ErrTooManyRequests = errors.New("too many requests") + // ErrOpenState is returned when the CB state is open + ErrOpenState = errors.New("circuit breaker is open") +) + +// String implements stringer interface. +func (s State) String() string { + switch s { + case StateClosed: + return "closed" + case StateHalfOpen: + return "half-open" + case StateOpen: + return "open" + default: + return fmt.Sprintf("unknown state: %d", s) + } +} + +// Counts holds the numbers of requests and their successes/failures. +// CircuitBreaker clears the internal Counts either +// on the change of the state or at the closed-state intervals. +// Counts ignores the results of the requests sent before clearing. +type Counts struct { + Requests uint32 + TotalSuccesses uint32 + TotalFailures uint32 + ConsecutiveSuccesses uint32 + ConsecutiveFailures uint32 +} + +func (c *Counts) onRequest() { + c.Requests++ +} + +func (c *Counts) onSuccess() { + c.TotalSuccesses++ + c.ConsecutiveSuccesses++ + c.ConsecutiveFailures = 0 +} + +func (c *Counts) onFailure() { + c.TotalFailures++ + c.ConsecutiveFailures++ + c.ConsecutiveSuccesses = 0 +} + +func (c *Counts) clear() { + c.Requests = 0 + c.TotalSuccesses = 0 + c.TotalFailures = 0 + c.ConsecutiveSuccesses = 0 + c.ConsecutiveFailures = 0 +} + +// Settings configures CircuitBreaker: +// +// Name is the name of the CircuitBreaker. +// +// MaxRequests is the maximum number of requests allowed to pass through +// when the CircuitBreaker is half-open. +// If MaxRequests is 0, the CircuitBreaker allows only 1 request. +// +// Interval is the cyclic period of the closed state +// for the CircuitBreaker to clear the internal Counts. +// If Interval is 0, the CircuitBreaker doesn't clear internal Counts during the closed state. +// +// Timeout is the period of the open state, +// after which the state of the CircuitBreaker becomes half-open. +// If Timeout is 0, the timeout value of the CircuitBreaker is set to 60 seconds. +// +// ReadyToTrip is called with a copy of Counts whenever a request fails in the closed state. +// If ReadyToTrip returns true, the CircuitBreaker will be placed into the open state. +// If ReadyToTrip is nil, default ReadyToTrip is used. +// Default ReadyToTrip returns true when the number of consecutive failures is more than 5. +// +// OnStateChange is called whenever the state of the CircuitBreaker changes. +type Settings struct { + Name string + MaxRequests uint32 + Interval time.Duration + Timeout time.Duration + ReadyToTrip func(counts Counts) bool + OnStateChange func(name string, from State, to State) +} + +// CircuitBreaker is a state machine to prevent sending requests that are likely to fail. +type CircuitBreaker struct { + name string + maxRequests uint32 + interval time.Duration + timeout time.Duration + readyToTrip func(counts Counts) bool + onStateChange func(name string, from State, to State) + + mutex sync.Mutex + state State + generation uint64 + counts Counts + expiry time.Time +} + +// TwoStepCircuitBreaker is like CircuitBreaker but instead of surrounding a function +// with the breaker functionality, it only checks whether a request can proceed and +// expects the caller to report the outcome in a separate step using a callback. +type TwoStepCircuitBreaker struct { + cb *CircuitBreaker +} + +// NewCircuitBreaker returns a new CircuitBreaker configured with the given Settings. +func NewCircuitBreaker(st Settings) *CircuitBreaker { + cb := new(CircuitBreaker) + + cb.name = st.Name + cb.interval = st.Interval + cb.onStateChange = st.OnStateChange + + if st.MaxRequests == 0 { + cb.maxRequests = 1 + } else { + cb.maxRequests = st.MaxRequests + } + + if st.Timeout == 0 { + cb.timeout = defaultTimeout + } else { + cb.timeout = st.Timeout + } + + if st.ReadyToTrip == nil { + cb.readyToTrip = defaultReadyToTrip + } else { + cb.readyToTrip = st.ReadyToTrip + } + + cb.toNewGeneration(time.Now()) + + return cb +} + +// NewTwoStepCircuitBreaker returns a new TwoStepCircuitBreaker configured with the given Settings. +func NewTwoStepCircuitBreaker(st Settings) *TwoStepCircuitBreaker { + return &TwoStepCircuitBreaker{ + cb: NewCircuitBreaker(st), + } +} + +const defaultTimeout = time.Duration(60) * time.Second + +func defaultReadyToTrip(counts Counts) bool { + return counts.ConsecutiveFailures > 5 +} + +// Name returns the name of the CircuitBreaker. +func (cb *CircuitBreaker) Name() string { + return cb.name +} + +// State returns the current state of the CircuitBreaker. +func (cb *CircuitBreaker) State() State { + cb.mutex.Lock() + defer cb.mutex.Unlock() + + now := time.Now() + state, _ := cb.currentState(now) + return state +} + +// Execute runs the given request if the CircuitBreaker accepts it. +// Execute returns an error instantly if the CircuitBreaker rejects the request. +// Otherwise, Execute returns the result of the request. +// If a panic occurs in the request, the CircuitBreaker handles it as an error +// and causes the same panic again. +func (cb *CircuitBreaker) Execute(req func() (interface{}, error)) (interface{}, error) { + generation, err := cb.beforeRequest() + if err != nil { + return nil, err + } + + defer func() { + e := recover() + if e != nil { + cb.afterRequest(generation, false) + panic(e) + } + }() + + result, err := req() + cb.afterRequest(generation, err == nil) + return result, err +} + +// Name returns the name of the TwoStepCircuitBreaker. +func (tscb *TwoStepCircuitBreaker) Name() string { + return tscb.cb.Name() +} + +// State returns the current state of the TwoStepCircuitBreaker. +func (tscb *TwoStepCircuitBreaker) State() State { + return tscb.cb.State() +} + +// Allow checks if a new request can proceed. It returns a callback that should be used to +// register the success or failure in a separate step. If the circuit breaker doesn't allow +// requests, it returns an error. +func (tscb *TwoStepCircuitBreaker) Allow() (done func(success bool), err error) { + generation, err := tscb.cb.beforeRequest() + if err != nil { + return nil, err + } + + return func(success bool) { + tscb.cb.afterRequest(generation, success) + }, nil +} + +func (cb *CircuitBreaker) beforeRequest() (uint64, error) { + cb.mutex.Lock() + defer cb.mutex.Unlock() + + now := time.Now() + state, generation := cb.currentState(now) + + if state == StateOpen { + return generation, ErrOpenState + } else if state == StateHalfOpen && cb.counts.Requests >= cb.maxRequests { + return generation, ErrTooManyRequests + } + + cb.counts.onRequest() + return generation, nil +} + +func (cb *CircuitBreaker) afterRequest(before uint64, success bool) { + cb.mutex.Lock() + defer cb.mutex.Unlock() + + now := time.Now() + state, generation := cb.currentState(now) + if generation != before { + return + } + + if success { + cb.onSuccess(state, now) + } else { + cb.onFailure(state, now) + } +} + +func (cb *CircuitBreaker) onSuccess(state State, now time.Time) { + switch state { + case StateClosed: + cb.counts.onSuccess() + case StateHalfOpen: + cb.counts.onSuccess() + if cb.counts.ConsecutiveSuccesses >= cb.maxRequests { + cb.setState(StateClosed, now) + } + } +} + +func (cb *CircuitBreaker) onFailure(state State, now time.Time) { + switch state { + case StateClosed: + cb.counts.onFailure() + if cb.readyToTrip(cb.counts) { + cb.setState(StateOpen, now) + } + case StateHalfOpen: + cb.setState(StateOpen, now) + } +} + +func (cb *CircuitBreaker) currentState(now time.Time) (State, uint64) { + switch cb.state { + case StateClosed: + if !cb.expiry.IsZero() && cb.expiry.Before(now) { + cb.toNewGeneration(now) + } + case StateOpen: + if cb.expiry.Before(now) { + cb.setState(StateHalfOpen, now) + } + } + return cb.state, cb.generation +} + +func (cb *CircuitBreaker) setState(state State, now time.Time) { + if cb.state == state { + return + } + + prev := cb.state + cb.state = state + + cb.toNewGeneration(now) + + if cb.onStateChange != nil { + cb.onStateChange(cb.name, prev, state) + } +} + +func (cb *CircuitBreaker) toNewGeneration(now time.Time) { + cb.generation++ + cb.counts.clear() + + var zero time.Time + switch cb.state { + case StateClosed: + if cb.interval == 0 { + cb.expiry = zero + } else { + cb.expiry = now.Add(cb.interval) + } + case StateOpen: + cb.expiry = now.Add(cb.timeout) + default: // StateHalfOpen + cb.expiry = zero + } +}