Bug 1014125 - Bisection Base Patch. r=ahal
This commit is contained in:
Vaibhav Agrawal
210
testing/mochitest/bisection.py
Normal file
210
testing/mochitest/bisection.py
Normal file
@@ -0,0 +1,210 @@
|
||||
import os
|
||||
import math
|
||||
import mozinfo
|
||||
|
||||
class Bisect(object):
|
||||
"Class for creating, bisecting and summarizing for --bisect-chunk option."
|
||||
|
||||
def __init__(self, harness):
|
||||
super(Bisect, self).__init__()
|
||||
self.summary = []
|
||||
self.contents = {}
|
||||
self.testRoot = harness.testRoot
|
||||
self.testRootAbs = harness.testRootAbs
|
||||
|
||||
def setup(self, tests):
|
||||
"This method is used to initialize various variables that are required for test bisection"
|
||||
status = 0
|
||||
self.contents.clear()
|
||||
# We need totalTests key in contents for sanity check
|
||||
self.contents['totalTests'] = tests
|
||||
self.contents['tests'] = tests
|
||||
self.contents['loop'] = 0
|
||||
return status
|
||||
|
||||
def reset(self, expectedError, result):
|
||||
"This method is used to initialize self.expectedError and self.result for each loop in runtests."
|
||||
self.expectedError = expectedError
|
||||
self.result = result
|
||||
|
||||
def get_test_chunk(self, options, tests):
|
||||
"This method is used to return the chunk of test that is to be run"
|
||||
if not options.totalChunks or not options.thisChunk:
|
||||
return tests
|
||||
|
||||
# The logic here is same as chunkifyTests.js, we need this for bisecting tests.
|
||||
if options.chunkByDir:
|
||||
tests_by_dir = {}
|
||||
test_dirs = []
|
||||
for test in tests:
|
||||
directory = test.split("/")
|
||||
directory = directory[0:min(options.chunkByDir, len(directory)-1)]
|
||||
directory = "/".join(directory)
|
||||
|
||||
if not directory in tests_by_dir:
|
||||
tests_by_dir[directory] = [test]
|
||||
test_dirs.append(directory)
|
||||
else:
|
||||
tests_by_dir[directory].append(test)
|
||||
|
||||
tests_per_chunk = float(len(test_dirs)) / options.totalChunks
|
||||
start = int(round((options.thisChunk-1) * tests_per_chunk))
|
||||
end = int(round((options.thisChunk) * tests_per_chunk))
|
||||
test_dirs = test_dirs[start:end]
|
||||
return_tests = []
|
||||
for directory in test_dirs:
|
||||
return_tests += tests_by_dir[directory]
|
||||
|
||||
else:
|
||||
tests_per_chunk = float(len(tests)) / options.totalChunks
|
||||
start = int(round((options.thisChunk-1) * tests_per_chunk))
|
||||
end = int(round(options.thisChunk * tests_per_chunk))
|
||||
return_tests = tests[start:end]
|
||||
|
||||
options.totalChunks = None
|
||||
options.thisChunk = None
|
||||
options.chunkByDir = None
|
||||
return return_tests
|
||||
|
||||
def get_tests_for_bisection(self, options, tests):
|
||||
"Make a list of tests for bisection from a given list of tests"
|
||||
tests = self.get_test_chunk(options, tests)
|
||||
bisectlist = []
|
||||
for test in tests:
|
||||
bisectlist.append(test)
|
||||
if test.endswith(options.bisectChunk):
|
||||
break
|
||||
|
||||
return bisectlist
|
||||
|
||||
def pre_test(self, options, tests, status):
|
||||
"This method is used to call other methods for setting up variables and getting the list of tests for bisection."
|
||||
if options.bisectChunk == "default":
|
||||
return tests
|
||||
# The second condition in 'if' is required to verify that the failing test is the last one.
|
||||
elif 'loop' not in self.contents or not self.contents['tests'][-1].endswith(options.bisectChunk):
|
||||
tests = self.get_tests_for_bisection(options, tests)
|
||||
status = self.setup(tests)
|
||||
|
||||
return self.next_chunk_reverse(options, status)
|
||||
|
||||
def post_test(self, options, expectedError, result):
|
||||
"This method is used to call other methods to summarize results and check whether a sanity check is done or not."
|
||||
self.reset(expectedError, result)
|
||||
status = self.summarize_chunk(options)
|
||||
# Check whether sanity check has to be done. Also it is necessary to check whether options.bisectChunk is present
|
||||
# in self.expectedError as we do not want to run if it is "default".
|
||||
if status == -1 and options.bisectChunk in self.expectedError:
|
||||
# In case we have a debug build, we don't want to run a sanity check, will take too much time.
|
||||
if mozinfo.info['debug']:
|
||||
return status
|
||||
|
||||
testBleedThrough = self.contents['testsToRun'][0]
|
||||
tests = self.contents['totalTests']
|
||||
tests.remove(testBleedThrough)
|
||||
# To make sure that the failing test is dependent on some other test.
|
||||
if options.bisectChunk in testBleedThrough:
|
||||
return status
|
||||
|
||||
status = self.setup(tests)
|
||||
self.summary.append("Sanity Check:")
|
||||
|
||||
return status
|
||||
|
||||
def next_chunk_reverse(self, options, status):
|
||||
"This method is used to bisect the tests in a reverse search fashion."
|
||||
|
||||
# Base Cases.
|
||||
if self.contents['loop'] == 0:
|
||||
self.contents['loop'] += 1
|
||||
self.contents['testsToRun'] = self.contents['tests']
|
||||
return self.contents['testsToRun']
|
||||
if self.contents['loop'] == 1:
|
||||
self.contents['loop'] += 1
|
||||
self.contents['testsToRun'] = [self.contents['tests'][-1]]
|
||||
return self.contents['testsToRun']
|
||||
|
||||
if 'result' in self.contents:
|
||||
if self.contents['result'] == "PASS":
|
||||
chunkSize = self.contents['end'] - self.contents['start']
|
||||
self.contents['end'] = self.contents['start'] - 1
|
||||
self.contents['start'] = self.contents['end'] - chunkSize
|
||||
|
||||
# self.contents['result'] will be expected error only if it fails.
|
||||
elif self.contents['result'] == "FAIL":
|
||||
self.contents['tests'] = self.contents['testsToRun']
|
||||
status = 1 # for initializing
|
||||
|
||||
# initialize
|
||||
if status:
|
||||
totalTests = len(self.contents['tests'])
|
||||
chunkSize = int(math.ceil(totalTests / 10.0))
|
||||
self.contents['start'] = totalTests - chunkSize - 1
|
||||
self.contents['end'] = totalTests - 2
|
||||
|
||||
start = self.contents['start']
|
||||
end = self.contents['end'] + 1
|
||||
self.contents['testsToRun'] = self.contents['tests'][start:end]
|
||||
self.contents['testsToRun'].append(self.contents['tests'][-1])
|
||||
self.contents['loop'] += 1
|
||||
|
||||
return self.contents['testsToRun']
|
||||
|
||||
def summarize_chunk(self, options):
|
||||
"This method is used summarize the results after the list of tests is run."
|
||||
if options.bisectChunk == "default":
|
||||
# if no expectedError that means all the tests have successfully passed.
|
||||
if len(self.expectedError) == 0:
|
||||
return -1
|
||||
options.bisectChunk = self.expectedError.keys()[0]
|
||||
self.summary.append("\tFound Error in test: %s" % options.bisectChunk)
|
||||
return 0
|
||||
|
||||
# If options.bisectChunk is not in self.result then we need to move to the next run.
|
||||
if options.bisectChunk not in self.result:
|
||||
return -1
|
||||
|
||||
self.summary.append("\tPass %d:" % self.contents['loop'])
|
||||
if len(self.contents['testsToRun']) > 1:
|
||||
self.summary.append("\t\t%d test files(start,end,failing). [%s, %s, %s]" % (len(self.contents['testsToRun']), self.contents['testsToRun'][0], self.contents['testsToRun'][-2], self.contents['testsToRun'][-1]))
|
||||
else:
|
||||
self.summary.append("\t\t1 test file [%s]" % self.contents['testsToRun'][0])
|
||||
|
||||
if self.result[options.bisectChunk] == "PASS":
|
||||
self.summary.append("\t\tno failures found.")
|
||||
if self.contents['loop'] == 1:
|
||||
status = -1
|
||||
elif self.contents['loop'] == 2:
|
||||
status = 1
|
||||
else:
|
||||
self.contents['result'] = "PASS"
|
||||
status = 0
|
||||
|
||||
elif self.result[options.bisectChunk] == "FAIL":
|
||||
if 'expectedError' not in self.contents:
|
||||
self.summary.append("\t\t%s failed." % self.contents['testsToRun'][-1])
|
||||
self.contents['expectedError'] = self.expectedError[options.bisectChunk]
|
||||
status = 0
|
||||
|
||||
elif self.expectedError[options.bisectChunk] == self.contents['expectedError']:
|
||||
self.summary.append("\t\t%s failed with expected error." % self.contents['testsToRun'][-1])
|
||||
self.contents['result'] = "FAIL"
|
||||
status = 0
|
||||
|
||||
# This code checks for test-bleedthrough. Should work for any algorithm.
|
||||
numberOfTests = len(self.contents['testsToRun'])
|
||||
if numberOfTests < 3:
|
||||
# This means that only 2 tests are run. Since the last test is the failing test itself therefore the bleedthrough test is the first test
|
||||
self.summary.append("TEST-BLEEDTHROUGH - found failure, %s" % self.contents['testsToRun'][0])
|
||||
status = -1
|
||||
else:
|
||||
self.summary.append("\t\t%s failed with different error." % self.contents['testsToRun'][-1])
|
||||
status = -1
|
||||
|
||||
return status
|
||||
|
||||
def print_summary(self):
|
||||
"This method is used to print the recorded summary."
|
||||
print "Bisection summary:"
|
||||
for line in self.summary:
|
||||
print line
|
||||
Reference in New Issue
Block a user