Would you consider giving the user a way to change this?
Our test suite has a persistent, frustrating race condition on login that causes tests to fail about 8-12% of the time. We are looking into fixing it (for real, I promise), but it would be very nice if we could just raise the cut-off, so that the webdriver only reports a failure if more than, say 15% of the tests failed.
Looking at your product, I see that the "siesta-launcher-all.js" file is the place that has this value hardcoded, in the "reviseFailedTests" function:
reviseFailedTests : function () {
var totalCount = 0
var failedCount = 0
this.forEachTestElement(function (el, group) {
totalCount++
var result = el.result
// don't count tests that finalized with ERROR
if (!result.ERROR && !result.passed) failedCount++
})
if (failedCount <= Math.max(totalCount * 0.1, 1))
this.forEachTestElement(function (el, group) {
var result = el.result
if (!result.ERROR && !result.passed && el.canRunAgain()) el.setProcessed(false)
})
}
Thank you,
David