| // Copyright 2020 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package breaker |
| |
| import ( |
| "sync" |
| "testing" |
| "time" |
| |
| "github.com/google/go-cmp/cmp" |
| "github.com/google/go-cmp/cmp/cmpopts" |
| ) |
| |
| func TestBucketReset(t *testing.T) { |
| bu := bucket{12, 8} |
| bu.reset() |
| if bu.successes != 0 { |
| t.Errorf("got successes = %d, want %d", bu.successes, 0) |
| } |
| if bu.failures != 0 { |
| t.Errorf("got failures = %d, want %d", bu.failures, 0) |
| } |
| } |
| |
| func TestResetCounts(t *testing.T) { |
| b := newTestBreaker(Config{}) |
| for i := 0; i < len(b.buckets); i++ { |
| b.buckets[i].successes = 10 |
| b.buckets[i].failures = 15 |
| } |
| |
| b.resetCounts() |
| testBuckets(t, b.buckets[:], 0, 0) |
| testCounts(t, b, 0, 0, 0) |
| } |
| |
| func TestNewBreaker(t *testing.T) { |
| timeNow = func() time.Time { |
| return time.Date(2020, time.May, 26, 18, 0, 0, 0, time.UTC) |
| } |
| got, err := New(Config{ |
| FailsToRed: 10, |
| FailureThreshold: 0.65, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: 30 * time.Second, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: 15, |
| }) |
| if err != nil { |
| t.Fatalf("New() returned %e, want nil", err) |
| } |
| want := &Breaker{ |
| config: Config{ |
| FailsToRed: 10, |
| FailureThreshold: 0.65, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: 30 * time.Second, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: 15, |
| }, |
| buckets: [numBuckets]bucket{}, |
| granularity: 2500 * time.Millisecond, |
| state: Green, |
| cur: 0, |
| consecutiveSuccs: 0, |
| timeout: 30 * time.Second, |
| lastEvent: time.Date(2020, time.May, 26, 18, 0, 0, 0, time.UTC), |
| } |
| |
| diff := cmp.Diff(want, got, cmpopts.IgnoreUnexported(sync.Mutex{}), cmp.AllowUnexported(Breaker{}, bucket{})) |
| if diff != "" { |
| t.Fatalf("mismatch (-want +got):\n%s", diff) |
| } |
| } |
| |
| func TestIllegalBreaker(t *testing.T) { |
| for _, test := range []struct { |
| name string |
| config Config |
| }{ |
| { |
| name: "FailsToRed cannot be 0", |
| config: Config{ |
| FailsToRed: 0, |
| FailureThreshold: 0.65, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: 30 * time.Second, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: 15, |
| }, |
| }, |
| { |
| name: "FailsToRed cannot be negative", |
| config: Config{ |
| FailsToRed: -5, |
| FailureThreshold: 0.65, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: 30 * time.Second, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: 15, |
| }, |
| }, |
| { |
| name: "FailureThreshold cannot be 0", |
| config: Config{ |
| FailsToRed: 8, |
| FailureThreshold: 0, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: 30 * time.Second, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: 15, |
| }, |
| }, |
| { |
| name: "FailureThreshold cannot be negative", |
| config: Config{ |
| FailsToRed: 8, |
| FailureThreshold: -0.8, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: 30 * time.Second, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: 15, |
| }, |
| }, |
| { |
| name: "FailureThreshold cannot exceed 1", |
| config: Config{ |
| FailsToRed: 8, |
| FailureThreshold: 1.2, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: 30 * time.Second, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: 15, |
| }, |
| }, |
| { |
| name: "GreenInterval cannot be 0", |
| config: Config{ |
| FailsToRed: 8, |
| FailureThreshold: 0.65, |
| GreenInterval: 0, |
| MinTimeout: 30 * time.Second, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: 15, |
| }, |
| }, |
| { |
| name: "GreenInterval cannot be negative", |
| config: Config{ |
| FailsToRed: 8, |
| FailureThreshold: 0.65, |
| GreenInterval: -4 * time.Second, |
| MinTimeout: 30 * time.Second, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: 15, |
| }, |
| }, |
| { |
| name: "MinTimeout cannot be 0", |
| config: Config{ |
| FailsToRed: 8, |
| FailureThreshold: 0.65, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: 0, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: 15, |
| }, |
| }, |
| { |
| name: "MinTimeout cannot be negative", |
| config: Config{ |
| FailsToRed: 8, |
| FailureThreshold: 0.65, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: -2 * time.Second, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: 15, |
| }, |
| }, |
| { |
| name: "MaxTimeout cannot be 0", |
| config: Config{ |
| FailsToRed: 8, |
| FailureThreshold: 0.65, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: 30 * time.Second, |
| MaxTimeout: 0, |
| SuccsToGreen: 15, |
| }, |
| }, |
| { |
| name: "MaxTimeout cannot be negative", |
| config: Config{ |
| FailsToRed: 8, |
| FailureThreshold: 0.65, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: 30 * time.Second, |
| MaxTimeout: -12 * time.Minute, |
| SuccsToGreen: 15, |
| }, |
| }, |
| { |
| name: "SuccsToGreen cannot be 0", |
| config: Config{ |
| FailsToRed: 8, |
| FailureThreshold: 0.65, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: 30 * time.Second, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: 0, |
| }, |
| }, |
| { |
| name: "SuccsToGreen cannot be negative", |
| config: Config{ |
| FailsToRed: 8, |
| FailureThreshold: 0.65, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: 30 * time.Second, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: -7, |
| }, |
| }, |
| { |
| name: "multiple illegal values return error", |
| config: Config{ |
| FailsToRed: 0, |
| FailureThreshold: 1.4, |
| GreenInterval: 20 * time.Second, |
| MinTimeout: -30 * time.Second, |
| MaxTimeout: 16 * time.Minute, |
| SuccsToGreen: 100, |
| }, |
| }, |
| } { |
| t.Run(test.name, func(t *testing.T) { |
| b, err := New(test.config) |
| if err == nil { |
| t.Fatalf("New() returned nil error") |
| } |
| if b != nil { |
| t.Fatalf("New() returned %+v, want nil", b) |
| } |
| }) |
| } |
| } |
| |
| func TestBreakerGranularity(t *testing.T) { |
| for _, test := range []struct { |
| config Config |
| want time.Duration |
| }{ |
| { |
| config: Config{}, |
| want: 1250 * time.Millisecond, |
| }, |
| { |
| config: Config{GreenInterval: 1 * time.Second}, |
| want: 125 * time.Millisecond, |
| }, |
| { |
| config: Config{GreenInterval: 3 * time.Second}, |
| want: 375 * time.Millisecond, |
| }, |
| { |
| config: Config{GreenInterval: 1 * time.Minute}, |
| want: 7500 * time.Millisecond, |
| }, |
| { |
| config: Config{GreenInterval: 1 * time.Hour}, |
| want: 450 * time.Second, |
| }, |
| } { |
| b := newTestBreaker(test.config) |
| if b.granularity != test.want { |
| t.Errorf("b.granularity = %d, want %d", b.granularity, test.want) |
| } |
| } |
| } |
| |
| func TestState(t *testing.T) { |
| for _, want := range []State{ |
| Green, |
| Yellow, |
| Red, |
| } { |
| b := newTestBreaker(Config{}) |
| b.state = want |
| if got := b.checkState(); got != want { |
| t.Errorf("b.checkState() = %s, got %s", got, want) |
| } |
| if got := b.State(); got != want { |
| t.Errorf("b.State() = %s, want %s", got, want) |
| } |
| } |
| } |
| func TestAllow(t *testing.T) { |
| for _, test := range []struct { |
| state State |
| shouldAllow bool |
| }{ |
| { |
| state: Green, |
| shouldAllow: true, |
| }, |
| { |
| state: Yellow, |
| shouldAllow: true, |
| }, |
| { |
| state: Red, |
| shouldAllow: false, |
| }, |
| } { |
| b := newTestBreaker(Config{}) |
| b.state = test.state |
| allowed := b.Allow() |
| if allowed != test.shouldAllow { |
| t.Errorf("b.Allow() = %t in %s, want %t", allowed, test.state, test.shouldAllow) |
| } |
| } |
| } |
| |
| func TestSuccesses(t *testing.T) { |
| b := newTestBreaker(Config{}) |
| b.succeeded() |
| testCounts(t, b, 1, 1, 0) |
| b.succeeded() |
| testCounts(t, b, 2, 2, 0) |
| b.succeeded() |
| testCounts(t, b, 3, 3, 0) |
| } |
| |
| func TestFailures(t *testing.T) { |
| b := newTestBreaker(Config{}) |
| b.failed() |
| testCounts(t, b, 0, 0, 1) |
| b.failed() |
| testCounts(t, b, 0, 0, 2) |
| b.failed() |
| testCounts(t, b, 0, 0, 3) |
| } |
| |
| func TestSucceededAndFailed(t *testing.T) { |
| b := newTestBreaker(Config{}) |
| b.succeeded() |
| testCounts(t, b, 1, 1, 0) |
| b.failed() |
| testCounts(t, b, 0, 1, 1) |
| b.failed() |
| testCounts(t, b, 0, 1, 2) |
| b.succeeded() |
| testCounts(t, b, 1, 2, 2) |
| b.succeeded() |
| testCounts(t, b, 2, 3, 2) |
| b.succeeded() |
| testCounts(t, b, 3, 4, 2) |
| b.failed() |
| testCounts(t, b, 0, 4, 3) |
| } |
| |
| func TestUpdate(t *testing.T) { |
| now := time.Now() |
| b := newTestBreaker(Config{}) |
| b.lastEvent = now |
| b.granularity = 1 * time.Second |
| for i := 0; i < len(b.buckets); i++ { |
| b.buckets[i].successes = 4 |
| b.buckets[i].failures = 9 |
| } |
| |
| // Update 0 buckets. |
| b.update(now.Add(-1 * time.Second)) |
| if b.cur != 0 { |
| t.Errorf("cur: got %d, want %d", b.cur, 0) |
| } |
| testBuckets(t, b.buckets[:], 4, 9) |
| testCounts(t, b, 0, 4*len(b.buckets), 9*len(b.buckets)) |
| |
| // Update next 3 buckets. |
| b.update(now.Add(3 * time.Second)) |
| if b.cur != 3 { |
| t.Errorf("cur: got %d, want %d", b.cur, 3) |
| } |
| testBuckets(t, b.buckets[:1], 4, 9) |
| testBuckets(t, b.buckets[1:4], 0, 0) |
| testBuckets(t, b.buckets[4:], 4, 9) |
| testCounts(t, b, 0, 4*len(b.buckets)-12, 9*len(b.buckets)-27) |
| |
| // Update all buckets. |
| b.update(now.Add(1003 * time.Second)) |
| expectedCur := 0 |
| if b.cur != expectedCur { |
| t.Errorf("cur: got %d, want %d", b.cur, expectedCur) |
| } |
| testBuckets(t, b.buckets[:], 0, 0) |
| testCounts(t, b, 0, 0, 0) |
| } |
| |
| func TestStateChanges(t *testing.T) { |
| for _, test := range []struct { |
| name string |
| config Config |
| preSuccesses int |
| preFailures int |
| fromState State |
| allow bool |
| success bool |
| sleep time.Duration |
| toState State |
| }{ |
| { |
| name: "breaker state remains green when FailsToRed is not exceeded", |
| config: Config{FailsToRed: 8, FailureThreshold: 0.5}, |
| preSuccesses: 6, |
| preFailures: 7, |
| fromState: Green, |
| allow: true, |
| success: false, |
| toState: Green, |
| }, |
| { |
| name: "breaker state remains green when FailureThreshold is not exceeded", |
| config: Config{FailsToRed: 2, FailureThreshold: 0.8}, |
| preSuccesses: 3, |
| preFailures: 6, |
| fromState: Green, |
| allow: true, |
| success: false, |
| toState: Green, |
| }, |
| { |
| name: "breaker state remains green when failure ratio = FailureThreshold", |
| config: Config{FailsToRed: 10, FailureThreshold: 0.5}, |
| preSuccesses: 20, |
| preFailures: 19, |
| fromState: Green, |
| allow: true, |
| success: false, |
| toState: Green, |
| }, |
| { |
| name: "breaker state remains green when failures = FailsToRed", |
| config: Config{FailsToRed: 10, FailureThreshold: 0.3}, |
| preSuccesses: 10, |
| preFailures: 9, |
| fromState: Green, |
| allow: true, |
| success: false, |
| toState: Green, |
| }, |
| { |
| name: "breaker state changes to red when FailureThreshold is exceeded and after FailsToRed has been exceeded", |
| config: Config{FailsToRed: 10, FailureThreshold: 0.5}, |
| preSuccesses: 20, |
| preFailures: 20, |
| fromState: Green, |
| allow: true, |
| success: false, |
| toState: Red, |
| }, |
| { |
| name: "breaker state changes to red when FailsToRed is exceeded and after FailureThreshold has been exceeded", |
| config: Config{FailsToRed: 20, FailureThreshold: 0.3}, |
| preSuccesses: 20, |
| preFailures: 20, |
| fromState: Green, |
| allow: true, |
| success: false, |
| toState: Red, |
| }, |
| { |
| name: "breaker state changes from green to red", |
| config: Config{FailsToRed: 4, FailureThreshold: 0.5}, |
| preSuccesses: 4, |
| preFailures: 4, |
| fromState: Green, |
| allow: true, |
| success: false, |
| toState: Red, |
| }, |
| { |
| name: "failure in yellow state changes breaker to red state", |
| config: Config{}, |
| preSuccesses: 0, |
| preFailures: 0, |
| fromState: Yellow, |
| allow: true, |
| success: false, |
| toState: Red, |
| }, |
| { |
| name: "breaker state changes from yellow to green", |
| config: Config{SuccsToGreen: 1}, |
| preSuccesses: 0, |
| preFailures: 0, |
| fromState: Yellow, |
| allow: true, |
| success: true, |
| toState: Green, |
| }, |
| { |
| name: "breaker state changes from red to yellow", |
| config: Config{MinTimeout: 1 * time.Second}, |
| preSuccesses: 0, |
| preFailures: 0, |
| fromState: Red, |
| sleep: 1*time.Second + 1*time.Nanosecond, |
| toState: Yellow, |
| }, |
| } { |
| t.Run(test.name, func(t *testing.T) { |
| now := time.Time{} |
| timeNow = func() time.Time { return now } |
| b := newTestBreaker(test.config) |
| b.state = test.fromState |
| b.buckets[0].successes = test.preSuccesses |
| b.buckets[0].failures = test.preFailures |
| |
| allowed := b.Allow() |
| if allowed != test.allow { |
| t.Fatalf("b.Allow() = %t in %s, want %t", allowed, test.fromState, test.allow) |
| } |
| if test.allow { |
| b.Record(test.success) |
| } |
| |
| // Pseudo sleep. |
| now = now.Add(test.sleep) |
| |
| if state := b.State(); state != test.toState { |
| t.Errorf("b.State() = %s, want %s", state, test.toState) |
| } |
| }) |
| } |
| } |
| |
| func TestRunningBreaker(t *testing.T) { |
| now := time.Time{} |
| timeNow = func() time.Time { return now } |
| b := newTestBreaker(Config{ |
| GreenInterval: 5 * time.Second, |
| }) |
| |
| // The following tests happen sequentially. The tests' states depend on previous tests. |
| for _, test := range []struct { |
| name string |
| firstSleep time.Duration |
| allow bool |
| secondSleep time.Duration |
| success bool |
| wantConsecutiveSuccs int |
| wantSuccesses int |
| wantFailures int |
| }{ |
| { |
| name: "successFunc called after a long time updates counts", |
| firstSleep: 20 * time.Second, |
| allow: true, |
| secondSleep: 20 * time.Second, |
| success: true, |
| wantConsecutiveSuccs: 1, |
| wantSuccesses: 1, |
| wantFailures: 0, |
| }, |
| { |
| name: "success within GreenInterval updates counts correctly", |
| firstSleep: 1 * time.Second, |
| allow: true, |
| secondSleep: 3 * time.Second, |
| success: true, |
| wantConsecutiveSuccs: 2, |
| wantSuccesses: 2, |
| wantFailures: 0, |
| }, |
| { |
| name: "success after a long time updates counts correctly", |
| firstSleep: 30 * time.Second, |
| allow: true, |
| secondSleep: 80 * time.Second, |
| success: true, |
| wantConsecutiveSuccs: 3, |
| wantSuccesses: 1, |
| wantFailures: 0, |
| }, |
| { |
| name: "failure within GreenInterval updates counts correctly", |
| firstSleep: 1 * time.Second, |
| allow: true, |
| secondSleep: 3 * time.Second, |
| success: false, |
| wantConsecutiveSuccs: 0, |
| wantSuccesses: 1, |
| wantFailures: 1, |
| }, |
| { |
| name: "second failure within GreenInterval updates counts correctly", |
| firstSleep: 1 * time.Millisecond, |
| allow: true, |
| secondSleep: 3 * time.Millisecond, |
| success: false, |
| wantConsecutiveSuccs: 0, |
| wantSuccesses: 1, |
| wantFailures: 2, |
| }, |
| { |
| name: "failure after a long time updates counts correctly", |
| firstSleep: 10 * time.Second, |
| allow: true, |
| secondSleep: 4 * time.Minute, |
| success: false, |
| wantConsecutiveSuccs: 0, |
| wantSuccesses: 0, |
| wantFailures: 1, |
| }, |
| } { |
| t.Run(test.name, func(t *testing.T) { |
| now = now.Add(test.firstSleep) |
| allowed := b.Allow() |
| |
| if allowed != test.allow { |
| t.Fatalf("breaker.Allow() = %t, want %t", allowed, test.allow) |
| } |
| |
| now = now.Add(test.secondSleep) |
| if test.allow { |
| b.Record(test.success) |
| } |
| |
| testCounts(t, b, test.wantConsecutiveSuccs, test.wantSuccesses, test.wantFailures) |
| }) |
| } |
| } |
| |
| func TestIncreaseTimeout(t *testing.T) { |
| b := newTestBreaker(Config{ |
| MinTimeout: 1 * time.Second, |
| MaxTimeout: 12 * time.Second, |
| }) |
| b.timeout = 3 * time.Second |
| |
| b.increaseTimeout() |
| testTimeouts(t, b, 6*time.Second, 1*time.Second, 12*time.Second) |
| b.increaseTimeout() |
| testTimeouts(t, b, 12*time.Second, 1*time.Second, 12*time.Second) |
| b.increaseTimeout() |
| testTimeouts(t, b, 12*time.Second, 1*time.Second, 12*time.Second) |
| |
| b.config.MaxTimeout = 14 * time.Second |
| testTimeouts(t, b, 12*time.Second, 1*time.Second, 14*time.Second) |
| b.increaseTimeout() |
| testTimeouts(t, b, 14*time.Second, 1*time.Second, 14*time.Second) |
| b.increaseTimeout() |
| testTimeouts(t, b, 14*time.Second, 1*time.Second, 14*time.Second) |
| } |
| |
| // newTestBreaker is like New, but with default values for easier testing. |
| func newTestBreaker(config Config) *Breaker { |
| if config.FailsToRed <= 0 { |
| config.FailsToRed = 10 |
| } |
| if config.FailureThreshold <= 0 { |
| config.FailureThreshold = 0.5 |
| } |
| if config.GreenInterval <= 0 { |
| config.GreenInterval = 10 * time.Second |
| } |
| if config.MinTimeout <= 0 { |
| config.MinTimeout = 30 * time.Second |
| } |
| if config.MaxTimeout <= 0 { |
| config.MaxTimeout = 4 * time.Minute |
| } |
| if config.SuccsToGreen <= 0 { |
| config.SuccsToGreen = 20 |
| } |
| b, _ := New(config) |
| return b |
| } |
| |
| func testCounts(t *testing.T, b *Breaker, consecutiveSuccs, wantSuccesses, wantFailures int) { |
| if b.consecutiveSuccs != consecutiveSuccs { |
| t.Errorf("b.consecutiveSuccs = %d, want %d", b.consecutiveSuccs, consecutiveSuccs) |
| } |
| successes, failures := b.counts() |
| if successes != wantSuccesses { |
| t.Errorf("successes = %d, want %d", successes, wantSuccesses) |
| } |
| if failures != wantFailures { |
| t.Errorf("failures = %d, want %d", failures, wantFailures) |
| } |
| } |
| |
| func testBuckets(t *testing.T, buckets []bucket, successes, failures int) { |
| for i, bu := range buckets { |
| if bu.successes != successes { |
| t.Errorf("slice bucket %d successes: got %d, want %d", i, bu.successes, successes) |
| } |
| if bu.failures != failures { |
| t.Errorf("slice bucket %d failures: got %d, want %d", i, bu.failures, failures) |
| } |
| } |
| } |
| |
| func testTimeouts(t *testing.T, b *Breaker, timeout, minTimeout, maxTimeout time.Duration) { |
| if b.timeout != timeout { |
| t.Errorf("b.timeout = %s, want %s", b.timeout, timeout) |
| } |
| if b.config.MinTimeout != minTimeout { |
| t.Errorf("b.config.MinTimeout = %s, want %s", b.config.MinTimeout, minTimeout) |
| } |
| if b.config.MaxTimeout != maxTimeout { |
| t.Errorf("b.config.MaxTimeout = %s, want %s", b.config.MaxTimeout, maxTimeout) |
| } |
| } |
