| # Shuffle order of tests and benchmarks |
| |
| # Run tests |
| go test -v foo_test.go |
| ! stdout '-test.shuffle ' |
| stdout '(?s)TestOne(.*)TestTwo(.*)TestThree' |
| |
| go test -v -shuffle=off foo_test.go |
| ! stdout '-test.shuffle ' |
| stdout '(?s)TestOne(.*)TestTwo(.*)TestThree' |
| |
| go test -v -shuffle=42 foo_test.go |
| stdout '^-test.shuffle 42' |
| stdout '(?s)TestThree(.*)TestOne(.*)TestTwo' |
| |
| go test -v -shuffle=43 foo_test.go |
| stdout '^-test.shuffle 43' |
| stdout '(?s)TestThree(.*)TestTwo(.*)TestOne' |
| |
| go test -v -shuffle=44 foo_test.go |
| stdout '^-test.shuffle 44' |
| stdout '(?s)TestOne(.*)TestThree(.*)TestTwo' |
| |
| go test -v -shuffle=0 foo_test.go |
| stdout '^-test.shuffle 0' |
| stdout '(?s)TestTwo(.*)TestOne(.*)TestThree' |
| |
| go test -v -shuffle -1 foo_test.go |
| stdout '^-test.shuffle -1' |
| stdout '(?s)TestThree(.*)TestOne(.*)TestTwo' |
| |
| go test -v -shuffle=on foo_test.go |
| stdout '^-test.shuffle ' |
| stdout '(?s)=== RUN TestOne(.*)--- PASS: TestOne' |
| stdout '(?s)=== RUN TestTwo(.*)--- PASS: TestTwo' |
| stdout '(?s)=== RUN TestThree(.*)--- PASS: TestThree' |
| |
| |
| # Run tests and benchmarks |
| go test -v -bench=. foo_test.go |
| ! stdout '-test.shuffle ' |
| stdout '(?s)TestOne(.*)TestTwo(.*)TestThree(.*)BenchmarkOne(.*)BenchmarkTwo(.*)BenchmarkThree' |
| |
| go test -v -bench=. -shuffle=off foo_test.go |
| ! stdout '-test.shuffle ' |
| stdout '(?s)TestOne(.*)TestTwo(.*)TestThree(.*)BenchmarkOne(.*)BenchmarkTwo(.*)BenchmarkThree' |
| |
| go test -v -bench=. -shuffle=42 foo_test.go |
| stdout '^-test.shuffle 42' |
| stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo' |
| |
| go test -v -bench=. -shuffle=43 foo_test.go |
| stdout '^-test.shuffle 43' |
| stdout '(?s)TestThree(.*)TestTwo(.*)TestOne(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo' |
| |
| go test -v -bench=. -shuffle=44 foo_test.go |
| stdout '^-test.shuffle 44' |
| stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree' |
| |
| go test -v -bench=. -shuffle=0 foo_test.go |
| stdout '^-test.shuffle 0' |
| stdout '(?s)TestTwo(.*)TestOne(.*)TestThree(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo' |
| |
| go test -v -bench=. -shuffle -1 foo_test.go |
| stdout '^-test.shuffle -1' |
| stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)BenchmarkTwo' |
| |
| go test -v -bench=. -shuffle=on foo_test.go |
| stdout '^-test.shuffle ' |
| stdout '(?s)=== RUN TestOne(.*)--- PASS: TestOne' |
| stdout '(?s)=== RUN TestTwo(.*)--- PASS: TestTwo' |
| stdout '(?s)=== RUN TestThree(.*)--- PASS: TestThree' |
| stdout -count=2 'BenchmarkOne' |
| stdout -count=2 'BenchmarkTwo' |
| stdout -count=2 'BenchmarkThree' |
| |
| |
| # When running go test -count=N, each of the N runs distinct runs should maintain the same |
| # shuffled order of these tests. |
| go test -v -shuffle=43 -count=4 foo_test.go |
| stdout '^-test.shuffle 43' |
| stdout '(?s)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne' |
| |
| go test -v -bench=. -shuffle=44 -count=2 foo_test.go |
| stdout '^-test.shuffle 44' |
| stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)' |
| |
| |
| # The feature should work with test binaries as well |
| go test -c |
| exec ./m.test -test.shuffle=off |
| ! stdout '^-test.shuffle ' |
| |
| exec ./m.test -test.shuffle=on |
| stdout '^-test.shuffle ' |
| |
| exec ./m.test -test.v -test.bench=. -test.shuffle=0 foo_test.go |
| stdout '^-test.shuffle 0' |
| stdout '(?s)TestTwo(.*)TestOne(.*)TestThree(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo' |
| |
| exec ./m.test -test.v -test.bench=. -test.shuffle=123 foo_test.go |
| stdout '^-test.shuffle 123' |
| stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkThree(.*)BenchmarkTwo(.*)BenchmarkOne' |
| |
| exec ./m.test -test.v -test.bench=. -test.shuffle=-1 foo_test.go |
| stdout '^-test.shuffle -1' |
| stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)BenchmarkTwo' |
| |
| exec ./m.test -test.v -test.bench=. -test.shuffle=44 -test.count=2 foo_test.go |
| stdout '^-test.shuffle 44' |
| stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)' |
| |
| |
| # Negative testcases for invalid input |
| ! go test -shuffle -count=2 |
| stderr 'invalid value "-count=2" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "-count=2": invalid syntax' |
| |
| ! go test -shuffle= |
| stderr '(?s)invalid value "" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "": invalid syntax' |
| |
| ! go test -shuffle=' ' |
| stderr '(?s)invalid value " " for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing " ": invalid syntax' |
| |
| ! go test -shuffle=true |
| stderr 'invalid value "true" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "true": invalid syntax' |
| |
| ! go test -shuffle='abc' |
| stderr 'invalid value "abc" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "abc": invalid syntax' |
| |
| -- go.mod -- |
| module m |
| |
| go 1.16 |
| -- foo_test.go -- |
| package foo |
| |
| import "testing" |
| |
| func TestOne(t *testing.T) {} |
| func TestTwo(t *testing.T) {} |
| func TestThree(t *testing.T) {} |
| |
| func BenchmarkOne(b *testing.B) {} |
| func BenchmarkTwo(b *testing.B) {} |
| func BenchmarkThree(b *testing.B) {} |
| |
| -- foo.go -- |
| package foo |