blob: 2a28653de73497e5b2627b248c489b79ccd27c12 [file] [log] [blame]
 // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Significance tests. package benchstat import ( "errors" "golang.org/x/perf/internal/stats" ) // A DeltaTest compares the old and new metrics and returns the // expected probability that they are drawn from the same distribution. // // If a probability cannot be computed, the DeltaTest returns an // error explaining why. Common errors include ErrSamplesEqual // (all samples are equal), ErrSampleSize (there aren't enough samples), // and ErrZeroVariance (the sample has zero variance). // // As a special case, the missing test NoDeltaTest returns -1, nil. type DeltaTest func(old, new *Metrics) (float64, error) // Errors returned by DeltaTest. var ( ErrSamplesEqual = errors.New("all equal") ErrSampleSize = errors.New("too few samples") ErrZeroVariance = errors.New("zero variance") ) // NoDeltaTest applies no delta test; it returns -1, nil. func NoDeltaTest(old, new *Metrics) (pval float64, err error) { return -1, nil } // TTest is a DeltaTest using the two-sample Welch t-test. func TTest(old, new *Metrics) (pval float64, err error) { t, err := stats.TwoSampleWelchTTest(stats.Sample{Xs: old.RValues}, stats.Sample{Xs: new.RValues}, stats.LocationDiffers) if err != nil { return -1, convertErr(err) } return t.P, nil } // UTest is a DeltaTest using the Mann-Whitney U test. func UTest(old, new *Metrics) (pval float64, err error) { u, err := stats.MannWhitneyUTest(old.RValues, new.RValues, stats.LocationDiffers) if err != nil { return -1, convertErr(err) } return u.P, nil } // convertErr converts from the stats package's internal errors // to errors exported by this package and expected from // a DeltaTest. // Using different errors makes it possible for clients to use // package benchstat without access to the internal stats package, // and it also gives us a chance to use shorter error messages. func convertErr(err error) error { switch err { case stats.ErrZeroVariance: return ErrZeroVariance case stats.ErrSampleSize: return ErrSampleSize case stats.ErrSamplesEqual: return ErrSamplesEqual } return err }