Bug 1601383 - Re-structure 'mach test-info' code; r=jmaher
Move almost all test-info code out of mach_commands.py and into its own file and consolidate logging and imports; no functional changes. Differential Revision: https://phabricator.services.mozilla.com/D56064
This commit is contained in:
Geoff Brown
@@ -5,8 +5,6 @@
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import argparse
|
||||
import errno
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
@@ -649,9 +647,6 @@ class CramTest(MachCommandBase):
|
||||
return subprocess.call(cmd, cwd=self.topsrcdir)
|
||||
|
||||
|
||||
ACTIVEDATA_RECORD_LIMIT = 10000
|
||||
|
||||
|
||||
@CommandProvider
|
||||
class TestInfoCommand(MachCommandBase):
|
||||
from datetime import date, timedelta
|
||||
@@ -690,400 +685,14 @@ class TestInfoCommand(MachCommandBase):
|
||||
help='Retrieve and display related Bugzilla bugs.')
|
||||
@CommandArgument('--verbose', action='store_true',
|
||||
help='Enable debug logging.')
|
||||
def test_info_tests(self, **params):
|
||||
from mozbuild.base import MozbuildObject
|
||||
from mozfile import which
|
||||
def test_info_tests(self, test_names, branches, start, end,
|
||||
show_info, show_results, show_durations, show_tasks, show_bugs,
|
||||
verbose):
|
||||
import testinfo
|
||||
|
||||
self.branches = params['branches']
|
||||
self.start = params['start']
|
||||
self.end = params['end']
|
||||
self.show_info = params['show_info']
|
||||
self.show_results = params['show_results']
|
||||
self.show_durations = params['show_durations']
|
||||
self.show_tasks = params['show_tasks']
|
||||
self.show_bugs = params['show_bugs']
|
||||
self.verbose = params['verbose']
|
||||
|
||||
if (not self.show_info and
|
||||
not self.show_results and
|
||||
not self.show_durations and
|
||||
not self.show_tasks and
|
||||
not self.show_bugs):
|
||||
# by default, show everything
|
||||
self.show_info = True
|
||||
self.show_results = True
|
||||
self.show_durations = True
|
||||
self.show_tasks = True
|
||||
self.show_bugs = True
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
build_obj = MozbuildObject.from_environment(cwd=here)
|
||||
|
||||
self._hg = None
|
||||
if conditions.is_hg(build_obj):
|
||||
self._hg = which('hg')
|
||||
if not self._hg:
|
||||
raise OSError(errno.ENOENT, "Could not find 'hg' on PATH.")
|
||||
|
||||
self._git = None
|
||||
if conditions.is_git(build_obj):
|
||||
self._git = which('git')
|
||||
if not self._git:
|
||||
raise OSError(errno.ENOENT, "Could not find 'git' on PATH.")
|
||||
|
||||
for test_name in params['test_names']:
|
||||
print("===== %s =====" % test_name)
|
||||
self.test_name = test_name
|
||||
if len(self.test_name) < 6:
|
||||
print("'%s' is too short for a test name!" % self.test_name)
|
||||
continue
|
||||
self.set_test_name()
|
||||
if self.show_bugs:
|
||||
self.report_bugs()
|
||||
self.set_activedata_test_name()
|
||||
if self.show_results:
|
||||
self.report_test_results()
|
||||
if self.show_durations:
|
||||
self.report_test_durations()
|
||||
if self.show_tasks:
|
||||
self.report_test_tasks()
|
||||
|
||||
def find_in_hg_or_git(self, test_name):
|
||||
if self._hg:
|
||||
cmd = [self._hg, 'files', '-I', test_name]
|
||||
elif self._git:
|
||||
cmd = [self._git, 'ls-files', test_name]
|
||||
else:
|
||||
return None
|
||||
try:
|
||||
out = subprocess.check_output(cmd).splitlines()
|
||||
except subprocess.CalledProcessError:
|
||||
out = None
|
||||
return out
|
||||
|
||||
def set_test_name(self):
|
||||
# Generating a unified report for a specific test is complicated
|
||||
# by differences in the test name used in various data sources.
|
||||
# Consider:
|
||||
# - It is often convenient to request a report based only on
|
||||
# a short file name, rather than the full path;
|
||||
# - Bugs may be filed in bugzilla against a simple, short test
|
||||
# name or the full path to the test;
|
||||
# - In ActiveData, the full path is usually used, but sometimes
|
||||
# also includes additional path components outside of the
|
||||
# mercurial repo (common for reftests).
|
||||
# This function attempts to find appropriate names for different
|
||||
# queries based on the specified test name.
|
||||
|
||||
import posixpath
|
||||
|
||||
# full_test_name is full path to file in hg (or git)
|
||||
self.full_test_name = None
|
||||
out = self.find_in_hg_or_git(self.test_name)
|
||||
if out and len(out) == 1:
|
||||
self.full_test_name = out[0]
|
||||
elif out and len(out) > 1:
|
||||
print("Ambiguous test name specified. Found:")
|
||||
for line in out:
|
||||
print(line)
|
||||
else:
|
||||
out = self.find_in_hg_or_git('**/%s*' % self.test_name)
|
||||
if out and len(out) == 1:
|
||||
self.full_test_name = out[0]
|
||||
elif out and len(out) > 1:
|
||||
print("Ambiguous test name. Found:")
|
||||
for line in out:
|
||||
print(line)
|
||||
if self.full_test_name:
|
||||
self.full_test_name.replace(os.sep, posixpath.sep)
|
||||
print("Found %s in source control." % self.full_test_name)
|
||||
else:
|
||||
print("Unable to validate test name '%s'!" % self.test_name)
|
||||
self.full_test_name = self.test_name
|
||||
|
||||
# search for full_test_name in test manifests
|
||||
from moztest.resolve import TestResolver, TestManifestLoader
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader)
|
||||
relpath = self._wrap_path_argument(self.full_test_name).relpath()
|
||||
tests = list(resolver.resolve_tests(paths=[relpath]))
|
||||
if len(tests) == 1:
|
||||
relpath = self._wrap_path_argument(tests[0]['manifest']).relpath()
|
||||
print("%s found in manifest %s" % (self.full_test_name, relpath))
|
||||
if tests[0].get('flavor'):
|
||||
print(" flavor: %s" % tests[0]['flavor'])
|
||||
if tests[0].get('skip-if'):
|
||||
print(" skip-if: %s" % tests[0]['skip-if'])
|
||||
if tests[0].get('fail-if'):
|
||||
print(" fail-if: %s" % tests[0]['fail-if'])
|
||||
elif len(tests) == 0:
|
||||
print("%s not found in any test manifest!" % self.full_test_name)
|
||||
else:
|
||||
print("%s found in more than one manifest!" % self.full_test_name)
|
||||
|
||||
# short_name is full_test_name without path
|
||||
self.short_name = None
|
||||
name_idx = self.full_test_name.rfind('/')
|
||||
if name_idx > 0:
|
||||
self.short_name = self.full_test_name[name_idx + 1:]
|
||||
if self.short_name and self.short_name == self.test_name:
|
||||
self.short_name = None
|
||||
|
||||
if not (self.show_results or self.show_durations or self.show_tasks):
|
||||
# no need to determine ActiveData name if not querying
|
||||
return
|
||||
|
||||
def set_activedata_test_name(self):
|
||||
import re
|
||||
|
||||
# activedata_test_name is name in ActiveData
|
||||
self.activedata_test_name = None
|
||||
simple_names = [
|
||||
self.full_test_name,
|
||||
self.test_name,
|
||||
self.short_name
|
||||
]
|
||||
simple_names = [x for x in simple_names if x]
|
||||
searches = [
|
||||
{"in": {"result.test": simple_names}},
|
||||
]
|
||||
regex_names = [".*%s.*" % re.escape(x) for x in simple_names if x]
|
||||
for r in regex_names:
|
||||
searches.append({"regexp": {"result.test": r}})
|
||||
query = {
|
||||
"from": "unittest",
|
||||
"format": "list",
|
||||
"limit": 10,
|
||||
"groupby": ["result.test"],
|
||||
"where": {"and": [
|
||||
{"or": searches},
|
||||
{"in": {"build.branch": self.branches.split(',')}},
|
||||
{"gt": {"run.timestamp": {"date": self.start}}},
|
||||
{"lt": {"run.timestamp": {"date": self.end}}}
|
||||
]}
|
||||
}
|
||||
print("Querying ActiveData...") # Following query can take a long time
|
||||
data = self.submit(query)
|
||||
if data and len(data) > 0:
|
||||
self.activedata_test_name = [
|
||||
d['result']['test']
|
||||
for p in simple_names + regex_names
|
||||
for d in data
|
||||
if re.match(p + "$", d['result']['test'])
|
||||
][0] # first match is best match
|
||||
if self.activedata_test_name:
|
||||
print("Found records matching '%s' in ActiveData." %
|
||||
self.activedata_test_name)
|
||||
else:
|
||||
print("Unable to find matching records in ActiveData; using %s!" %
|
||||
self.test_name)
|
||||
self.activedata_test_name = self.test_name
|
||||
|
||||
def get_run_types(self, record):
|
||||
types_label = ""
|
||||
if 'run' in record and 'type' in record['run']:
|
||||
run_types = record['run']['type']
|
||||
run_types = run_types if isinstance(run_types, list) else [run_types]
|
||||
fission = True if 'fis' in run_types else False
|
||||
for run_type in run_types:
|
||||
# chunked is not interesting
|
||||
if run_type == 'chunked':
|
||||
continue
|
||||
# fission implies e10s
|
||||
if fission and run_type == 'e10s':
|
||||
continue
|
||||
types_label += "-" + run_type
|
||||
return types_label
|
||||
|
||||
def get_platform(self, record):
|
||||
if 'platform' in record['build']:
|
||||
platform = record['build']['platform']
|
||||
else:
|
||||
platform = "-"
|
||||
tp = record['build']['type']
|
||||
if type(tp) is list:
|
||||
tp = "-".join(tp)
|
||||
return "%s/%s%s:" % (platform, tp, self.get_run_types(record))
|
||||
|
||||
def submit(self, query):
|
||||
import datetime
|
||||
import requests
|
||||
if self.verbose:
|
||||
print(datetime.datetime.now())
|
||||
print(json.dumps(query))
|
||||
response = requests.post("http://activedata.allizom.org/query",
|
||||
data=json.dumps(query),
|
||||
stream=True)
|
||||
if self.verbose:
|
||||
print(datetime.datetime.now())
|
||||
print(response)
|
||||
response.raise_for_status()
|
||||
data = response.json()["data"]
|
||||
return data
|
||||
|
||||
def report_test_results(self):
|
||||
# Report test pass/fail summary from ActiveData
|
||||
query = {
|
||||
"from": "unittest",
|
||||
"format": "list",
|
||||
"limit": 100,
|
||||
"groupby": ["build.platform", "build.type", "run.type"],
|
||||
"select": [
|
||||
{"aggregate": "count"},
|
||||
{
|
||||
"name": "failures",
|
||||
"value": {"case": [
|
||||
{"when": {"eq": {"result.ok": "F"}}, "then": 1}
|
||||
]},
|
||||
"aggregate": "sum",
|
||||
"default": 0
|
||||
},
|
||||
{
|
||||
"name": "skips",
|
||||
"value": {"case": [
|
||||
{"when": {"eq": {"result.status": "SKIP"}}, "then": 1}
|
||||
]},
|
||||
"aggregate": "sum",
|
||||
"default": 0
|
||||
}
|
||||
],
|
||||
"where": {"and": [
|
||||
{"eq": {"result.test": self.activedata_test_name}},
|
||||
{"in": {"build.branch": self.branches.split(',')}},
|
||||
{"gt": {"run.timestamp": {"date": self.start}}},
|
||||
{"lt": {"run.timestamp": {"date": self.end}}}
|
||||
]}
|
||||
}
|
||||
print("\nTest results for %s on %s between %s and %s" %
|
||||
(self.activedata_test_name, self.branches, self.start, self.end))
|
||||
data = self.submit(query)
|
||||
if data and len(data) > 0:
|
||||
data.sort(key=self.get_platform)
|
||||
worst_rate = 0.0
|
||||
worst_platform = None
|
||||
total_runs = 0
|
||||
total_failures = 0
|
||||
for record in data:
|
||||
platform = self.get_platform(record)
|
||||
if platform.startswith("-"):
|
||||
continue
|
||||
runs = record['count']
|
||||
total_runs = total_runs + runs
|
||||
failures = record.get('failures', 0)
|
||||
skips = record.get('skips', 0)
|
||||
total_failures = total_failures + failures
|
||||
rate = (float)(failures) / runs
|
||||
if rate >= worst_rate:
|
||||
worst_rate = rate
|
||||
worst_platform = platform
|
||||
worst_failures = failures
|
||||
worst_runs = runs
|
||||
print("%-40s %6d failures (%6d skipped) in %6d runs" % (
|
||||
platform, failures, skips, runs))
|
||||
print("\nTotal: %d failures in %d runs or %.3f failures/run" %
|
||||
(total_failures, total_runs, (float)(total_failures) / total_runs))
|
||||
if worst_failures > 0:
|
||||
print("Worst rate on %s %d failures in %d runs or %.3f failures/run" %
|
||||
(worst_platform, worst_failures, worst_runs, worst_rate))
|
||||
else:
|
||||
print("No test result data found.")
|
||||
|
||||
def report_test_durations(self):
|
||||
# Report test durations summary from ActiveData
|
||||
query = {
|
||||
"from": "unittest",
|
||||
"format": "list",
|
||||
"limit": 100,
|
||||
"groupby": ["build.platform", "build.type", "run.type"],
|
||||
"select": [
|
||||
{"value": "result.duration",
|
||||
"aggregate": "average", "name": "average"},
|
||||
{"value": "result.duration", "aggregate": "min", "name": "min"},
|
||||
{"value": "result.duration", "aggregate": "max", "name": "max"},
|
||||
{"aggregate": "count"}
|
||||
],
|
||||
"where": {"and": [
|
||||
{"eq": {"result.ok": "T"}},
|
||||
{"eq": {"result.test": self.activedata_test_name}},
|
||||
{"in": {"build.branch": self.branches.split(',')}},
|
||||
{"gt": {"run.timestamp": {"date": self.start}}},
|
||||
{"lt": {"run.timestamp": {"date": self.end}}}
|
||||
]}
|
||||
}
|
||||
data = self.submit(query)
|
||||
print("\nTest durations for %s on %s between %s and %s" %
|
||||
(self.activedata_test_name, self.branches, self.start, self.end))
|
||||
if data and len(data) > 0:
|
||||
data.sort(key=self.get_platform)
|
||||
for record in data:
|
||||
platform = self.get_platform(record)
|
||||
if platform.startswith("-"):
|
||||
continue
|
||||
print("%-40s %6.2f s (%.2f s - %.2f s over %d runs)" % (
|
||||
platform, record['average'], record['min'],
|
||||
record['max'], record['count']))
|
||||
else:
|
||||
print("No test durations found.")
|
||||
|
||||
def report_test_tasks(self):
|
||||
# Report test tasks summary from ActiveData
|
||||
query = {
|
||||
"from": "unittest",
|
||||
"format": "list",
|
||||
"limit": 1000,
|
||||
"select": ["build.platform", "build.type", "run.type", "run.name"],
|
||||
"where": {"and": [
|
||||
{"eq": {"result.test": self.activedata_test_name}},
|
||||
{"in": {"build.branch": self.branches.split(',')}},
|
||||
{"gt": {"run.timestamp": {"date": self.start}}},
|
||||
{"lt": {"run.timestamp": {"date": self.end}}}
|
||||
]}
|
||||
}
|
||||
data = self.submit(query)
|
||||
print("\nTest tasks for %s on %s between %s and %s" %
|
||||
(self.activedata_test_name, self.branches, self.start, self.end))
|
||||
if data and len(data) > 0:
|
||||
data.sort(key=self.get_platform)
|
||||
consolidated = {}
|
||||
for record in data:
|
||||
platform = self.get_platform(record)
|
||||
if platform not in consolidated:
|
||||
consolidated[platform] = {}
|
||||
if record['run']['name'] in consolidated[platform]:
|
||||
consolidated[platform][record['run']['name']] += 1
|
||||
else:
|
||||
consolidated[platform][record['run']['name']] = 1
|
||||
for key in sorted(consolidated.keys()):
|
||||
tasks = ""
|
||||
for task in consolidated[key].keys():
|
||||
if tasks:
|
||||
tasks += "\n%-40s " % ""
|
||||
tasks += task
|
||||
tasks += " in %d runs" % consolidated[key][task]
|
||||
print("%-40s %s" % (key, tasks))
|
||||
else:
|
||||
print("No test tasks found.")
|
||||
|
||||
def report_bugs(self):
|
||||
# Report open bugs matching test name
|
||||
import requests
|
||||
search = self.full_test_name
|
||||
if self.test_name:
|
||||
search = '%s,%s' % (search, self.test_name)
|
||||
if self.short_name:
|
||||
search = '%s,%s' % (search, self.short_name)
|
||||
payload = {'quicksearch': search,
|
||||
'include_fields': 'id,summary'}
|
||||
response = requests.get('https://bugzilla.mozilla.org/rest/bug',
|
||||
payload)
|
||||
response.raise_for_status()
|
||||
json_response = response.json()
|
||||
print("\nBugzilla quick search for '%s':" % search)
|
||||
if 'bugs' in json_response:
|
||||
for bug in json_response['bugs']:
|
||||
print("Bug %s: %s" % (bug['id'], bug['summary']))
|
||||
else:
|
||||
print("No bugs found.")
|
||||
ti = testinfo.TestInfoTests(verbose)
|
||||
ti.report(test_names, branches, start, end,
|
||||
show_info, show_results, show_durations, show_tasks, show_bugs)
|
||||
|
||||
@SubCommand('test-info', 'long-tasks',
|
||||
description='Find tasks approaching their taskcluster max-run-time.')
|
||||
@@ -1106,80 +715,15 @@ class TestInfoCommand(MachCommandBase):
|
||||
help='Report tasks exceeding this percentage of long tasks.')
|
||||
@CommandArgument('--verbose', action='store_true',
|
||||
help='Enable debug logging.')
|
||||
def report_long_running_tasks(self, **params):
|
||||
def get_long_running_ratio(record):
|
||||
count = record['count']
|
||||
tasks_gt_pct = record['tasks_gt_pct']
|
||||
return count / tasks_gt_pct
|
||||
def report_long_running_tasks(self, branches, start, end,
|
||||
max_threshold_pct, filter_threshold_pct, verbose):
|
||||
import testinfo
|
||||
|
||||
branches = params['branches']
|
||||
start = params['start']
|
||||
end = params['end']
|
||||
self.verbose = params['verbose']
|
||||
threshold_pct = float(params['max_threshold_pct'])
|
||||
filter_threshold_pct = float(params['filter_threshold_pct'])
|
||||
max_threshold_pct = float(max_threshold_pct)
|
||||
filter_threshold_pct = float(filter_threshold_pct)
|
||||
|
||||
# Search test durations in ActiveData for long-running tests
|
||||
query = {
|
||||
"from": "task",
|
||||
"format": "list",
|
||||
"groupby": ["run.name"],
|
||||
"limit": 1000,
|
||||
"select": [
|
||||
{
|
||||
"value": "task.maxRunTime",
|
||||
"aggregate": "median",
|
||||
"name": "max_run_time"
|
||||
},
|
||||
{
|
||||
"aggregate": "count"
|
||||
},
|
||||
{
|
||||
"value": {
|
||||
"when": {
|
||||
"gt": [
|
||||
{
|
||||
"div": ["action.duration", "task.maxRunTime"]
|
||||
}, threshold_pct/100.0
|
||||
]
|
||||
},
|
||||
"then": 1
|
||||
},
|
||||
"aggregate": "sum",
|
||||
"name": "tasks_gt_pct"
|
||||
},
|
||||
],
|
||||
"where": {"and": [
|
||||
{"in": {"build.branch": branches.split(',')}},
|
||||
{"gt": {"task.run.start_time": {"date": start}}},
|
||||
{"lte": {"task.run.start_time": {"date": end}}},
|
||||
{"eq": {"task.state": "completed"}},
|
||||
]}
|
||||
}
|
||||
data = self.submit(query)
|
||||
print("\nTasks nearing their max-run-time on %s between %s and %s" %
|
||||
(branches, start, end))
|
||||
if data and len(data) > 0:
|
||||
filtered = []
|
||||
for record in data:
|
||||
if 'tasks_gt_pct' in record:
|
||||
count = record['count']
|
||||
tasks_gt_pct = record['tasks_gt_pct']
|
||||
if float(tasks_gt_pct) / count > filter_threshold_pct / 100.0:
|
||||
filtered.append(record)
|
||||
filtered.sort(key=get_long_running_ratio)
|
||||
if not filtered:
|
||||
print("No long running tasks found.")
|
||||
for record in filtered:
|
||||
name = record['run']['name']
|
||||
count = record['count']
|
||||
max_run_time = record['max_run_time']
|
||||
tasks_gt_pct = record['tasks_gt_pct']
|
||||
print("%-55s: %d of %d runs (%.1f%%) exceeded %d%% of max-run-time (%d s)" %
|
||||
(name, tasks_gt_pct, count, tasks_gt_pct * 100 / count,
|
||||
threshold_pct, max_run_time))
|
||||
else:
|
||||
print("No tasks found.")
|
||||
ti = testinfo.TestInfoLongRunningTasks(verbose)
|
||||
ti.report(branches, start, end, max_threshold_pct, filter_threshold_pct)
|
||||
|
||||
@SubCommand('test-info', 'report',
|
||||
description='Generate a json report of test manifests and/or tests '
|
||||
@@ -1224,49 +768,15 @@ class TestInfoCommand(MachCommandBase):
|
||||
type=int,
|
||||
default=7,
|
||||
help='Query ActiveData for specified number of days')
|
||||
@CommandArgument('--verbose', action='store_true',
|
||||
help='Enable debug logging.')
|
||||
def test_report(self, components, flavor, subsuite, paths,
|
||||
show_manifests, show_tests, show_summary, show_annotations,
|
||||
show_activedata,
|
||||
filter_values, filter_keys, show_components, output_file,
|
||||
branches, days):
|
||||
import re
|
||||
import traceback
|
||||
import mozpack.path as mozpath
|
||||
branches, days, verbose):
|
||||
import testinfo
|
||||
from mozbuild.build_commands import Build
|
||||
from moztest.resolve import TestResolver, TestManifestLoader
|
||||
|
||||
def matches_filters(test):
|
||||
'''
|
||||
Return True if all of the requested filter_values are found in this test;
|
||||
if filter_keys are specified, restrict search to those test keys.
|
||||
'''
|
||||
for value in filter_values:
|
||||
value_found = False
|
||||
for key in test:
|
||||
if not filter_keys or key in filter_keys:
|
||||
if re.search(value, test[key]):
|
||||
value_found = True
|
||||
break
|
||||
if not value_found:
|
||||
return False
|
||||
return True
|
||||
|
||||
# Ensure useful report by default
|
||||
if not show_manifests and not show_tests and not show_summary and not show_annotations:
|
||||
show_manifests = True
|
||||
show_summary = True
|
||||
|
||||
by_component = {}
|
||||
if components:
|
||||
components = components.split(',')
|
||||
if filter_keys:
|
||||
filter_keys = filter_keys.split(',')
|
||||
if filter_values:
|
||||
filter_values = filter_values.split(',')
|
||||
else:
|
||||
filter_values = []
|
||||
display_keys = (filter_keys or []) + ['skip-if', 'fail-if', 'fails-if']
|
||||
display_keys = set(display_keys)
|
||||
|
||||
try:
|
||||
self.config_environment
|
||||
@@ -1275,330 +785,12 @@ class TestInfoCommand(MachCommandBase):
|
||||
builder = Build(self._mach_context)
|
||||
builder.configure()
|
||||
|
||||
print("Finding tests...")
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader)
|
||||
tests = list(resolver.resolve_tests(paths=paths, flavor=flavor,
|
||||
subsuite=subsuite))
|
||||
|
||||
manifest_paths = set()
|
||||
for t in tests:
|
||||
if 'manifest' in t and t['manifest'] is not None:
|
||||
manifest_paths.add(t['manifest'])
|
||||
manifest_count = len(manifest_paths)
|
||||
print("Resolver found {} tests, {} manifests".format(len(tests), manifest_count))
|
||||
|
||||
if show_manifests:
|
||||
by_component['manifests'] = {}
|
||||
manifest_paths = list(manifest_paths)
|
||||
manifest_paths.sort()
|
||||
relpaths = []
|
||||
for manifest_path in manifest_paths:
|
||||
relpath = mozpath.relpath(manifest_path, self.topsrcdir)
|
||||
if mozpath.commonprefix((manifest_path, self.topsrcdir)) != self.topsrcdir:
|
||||
continue
|
||||
relpaths.append(relpath)
|
||||
reader = self.mozbuild_reader(config_mode='empty')
|
||||
files_info = reader.files_info(relpaths)
|
||||
for manifest_path in manifest_paths:
|
||||
relpath = mozpath.relpath(manifest_path, self.topsrcdir)
|
||||
if mozpath.commonprefix((manifest_path, self.topsrcdir)) != self.topsrcdir:
|
||||
continue
|
||||
manifest_info = None
|
||||
if relpath in files_info:
|
||||
bug_component = files_info[relpath].get('BUG_COMPONENT')
|
||||
key = "{}::{}".format(bug_component.product, bug_component.component)
|
||||
if (not components) or (key in components):
|
||||
manifest_info = {
|
||||
'manifest': relpath,
|
||||
'tests': 0,
|
||||
'skipped': 0
|
||||
}
|
||||
rkey = key if show_components else 'all'
|
||||
if rkey in by_component['manifests']:
|
||||
by_component['manifests'][rkey].append(manifest_info)
|
||||
else:
|
||||
by_component['manifests'][rkey] = [manifest_info]
|
||||
if manifest_info:
|
||||
for t in tests:
|
||||
if t['manifest'] == manifest_path:
|
||||
manifest_info['tests'] += 1
|
||||
if t.get('skip-if'):
|
||||
manifest_info['skipped'] += 1
|
||||
for key in by_component['manifests']:
|
||||
by_component['manifests'][key].sort()
|
||||
|
||||
if show_tests:
|
||||
by_component['tests'] = {}
|
||||
|
||||
if show_tests or show_summary or show_annotations:
|
||||
test_count = 0
|
||||
failed_count = 0
|
||||
skipped_count = 0
|
||||
annotation_count = 0
|
||||
condition_count = 0
|
||||
component_set = set()
|
||||
relpaths = []
|
||||
conditions = {}
|
||||
known_unconditional_annotations = ['skip', 'fail', 'asserts', 'random']
|
||||
known_conditional_annotations = ['skip-if', 'fail-if', 'run-if',
|
||||
'fails-if', 'fuzzy-if', 'random-if', 'asserts-if']
|
||||
for t in tests:
|
||||
relpath = t.get('srcdir_relpath')
|
||||
relpaths.append(relpath)
|
||||
reader = self.mozbuild_reader(config_mode='empty')
|
||||
files_info = reader.files_info(relpaths)
|
||||
for t in tests:
|
||||
if not matches_filters(t):
|
||||
continue
|
||||
if 'referenced-test' in t:
|
||||
# Avoid double-counting reftests: disregard reference file entries
|
||||
continue
|
||||
if show_annotations:
|
||||
for key in t:
|
||||
if key in known_unconditional_annotations:
|
||||
annotation_count += 1
|
||||
if key in known_conditional_annotations:
|
||||
annotation_count += 1
|
||||
# Here 'key' is a manifest annotation type like 'skip-if' and t[key]
|
||||
# is the associated condition. For example, the manifestparser
|
||||
# manifest annotation, "skip-if = os == 'win'", is expected to be
|
||||
# encoded as t['skip-if'] = "os == 'win'".
|
||||
# To allow for reftest manifests, t[key] may have multiple entries
|
||||
# separated by ';', each corresponding to a condition for that test
|
||||
# and annotation type. For example,
|
||||
# "skip-if(Android&&webrender) skip-if(OSX)", would be
|
||||
# encoded as t['skip-if'] = "Android&&webrender;OSX".
|
||||
annotation_conditions = t[key].split(';')
|
||||
for condition in annotation_conditions:
|
||||
condition_count += 1
|
||||
# Trim reftest fuzzy-if ranges: everything after the first comma
|
||||
# eg. "Android,0-2,1-3" -> "Android"
|
||||
condition = condition.split(',')[0]
|
||||
if condition not in conditions:
|
||||
conditions[condition] = 0
|
||||
conditions[condition] += 1
|
||||
test_count += 1
|
||||
relpath = t.get('srcdir_relpath')
|
||||
if relpath in files_info:
|
||||
bug_component = files_info[relpath].get('BUG_COMPONENT')
|
||||
key = "{}::{}".format(bug_component.product, bug_component.component)
|
||||
if (not components) or (key in components):
|
||||
component_set.add(key)
|
||||
test_info = {'test': relpath}
|
||||
for test_key in display_keys:
|
||||
value = t.get(test_key)
|
||||
if value:
|
||||
test_info[test_key] = value
|
||||
if t.get('fail-if'):
|
||||
failed_count += 1
|
||||
if t.get('fails-if'):
|
||||
failed_count += 1
|
||||
if t.get('skip-if'):
|
||||
skipped_count += 1
|
||||
if show_tests:
|
||||
rkey = key if show_components else 'all'
|
||||
if rkey in by_component['tests']:
|
||||
by_component['tests'][rkey].append(test_info)
|
||||
else:
|
||||
by_component['tests'][rkey] = [test_info]
|
||||
if show_tests:
|
||||
for key in by_component['tests']:
|
||||
by_component['tests'][key].sort(key=lambda k: k['test'])
|
||||
|
||||
if show_activedata:
|
||||
try:
|
||||
self.add_activedata(branches, days, by_component)
|
||||
except Exception:
|
||||
print("Failed to retrieve some ActiveData data.")
|
||||
traceback.print_exc()
|
||||
|
||||
if show_summary:
|
||||
by_component['summary'] = {}
|
||||
by_component['summary']['components'] = len(component_set)
|
||||
by_component['summary']['manifests'] = manifest_count
|
||||
by_component['summary']['tests'] = test_count
|
||||
by_component['summary']['failed tests'] = failed_count
|
||||
by_component['summary']['skipped tests'] = skipped_count
|
||||
|
||||
if show_annotations:
|
||||
by_component['annotations'] = {}
|
||||
by_component['annotations']['total annotations'] = annotation_count
|
||||
by_component['annotations']['total conditions'] = condition_count
|
||||
by_component['annotations']['unique conditions'] = len(conditions)
|
||||
by_component['annotations']['conditions'] = conditions
|
||||
|
||||
json_report = json.dumps(by_component, indent=2, sort_keys=True)
|
||||
if output_file:
|
||||
output_file = os.path.abspath(output_file)
|
||||
output_dir = os.path.dirname(output_file)
|
||||
if not os.path.isdir(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
|
||||
with open(output_file, 'w') as f:
|
||||
f.write(json_report)
|
||||
else:
|
||||
print(json_report)
|
||||
|
||||
def add_activedata_for_suite(self, branches, days, by_component,
|
||||
suite_clause, path_mod):
|
||||
import requests
|
||||
|
||||
def submit(query):
|
||||
response = requests.post("http://activedata.allizom.org/query",
|
||||
data=json.dumps(query),
|
||||
stream=True)
|
||||
response.raise_for_status()
|
||||
data = response.json()["data"]
|
||||
return data
|
||||
|
||||
dates_clause = {"date": "today-%dday" % days}
|
||||
ad_query = {
|
||||
"from": "unittest",
|
||||
"limit": ACTIVEDATA_RECORD_LIMIT,
|
||||
"format": "list",
|
||||
"groupby": ["result.test"],
|
||||
"select": [
|
||||
{
|
||||
"name": "result.count",
|
||||
"aggregate": "count"
|
||||
},
|
||||
{
|
||||
"name": "result.duration",
|
||||
"value": "result.duration",
|
||||
"aggregate": "sum"
|
||||
},
|
||||
{
|
||||
"name": "result.failures",
|
||||
"value": {"case": [
|
||||
{"when": {"eq": {"result.ok": "F"}}, "then": 1}
|
||||
]},
|
||||
"aggregate": "sum",
|
||||
"default": 0
|
||||
},
|
||||
{
|
||||
"name": "result.skips",
|
||||
"value": {"case": [
|
||||
{"when": {"eq": {"result.status": "SKIP"}}, "then": 1}
|
||||
]},
|
||||
"aggregate": "sum",
|
||||
"default": 0
|
||||
}
|
||||
],
|
||||
"where": {"and": [
|
||||
suite_clause,
|
||||
{"in": {"repo.branch.name": branches.split(',')}},
|
||||
{"gt": {"run.timestamp": dates_clause}}
|
||||
]}
|
||||
}
|
||||
ad_response = submit(ad_query)
|
||||
if len(ad_response) == ACTIVEDATA_RECORD_LIMIT:
|
||||
print("warning: limit reached; some data may be missing")
|
||||
for record in ad_response:
|
||||
if 'result' in record:
|
||||
result = record['result']
|
||||
self.update_report(by_component, result, path_mod)
|
||||
|
||||
def update_report(self, by_component, result, path_mod):
|
||||
if 'test' in result and 'tests' in by_component:
|
||||
test = result['test']
|
||||
if path_mod:
|
||||
test = path_mod(test)
|
||||
for bc in by_component['tests']:
|
||||
for item in by_component['tests'][bc]:
|
||||
if test == item['test']:
|
||||
seconds = result.get('duration', 'unknown')
|
||||
if seconds != 'unknown':
|
||||
seconds = round(seconds, 2)
|
||||
item['total run time, seconds'] = seconds
|
||||
item['total runs'] = result.get('count', 'unknown')
|
||||
item['skipped runs'] = result.get('skips', 'unknown')
|
||||
item['failed runs'] = result.get('failures', 'unknown')
|
||||
return True
|
||||
return False
|
||||
|
||||
def path_mod_reftest(self, path):
|
||||
# "<path1> == <path2>" -> "<path1>"
|
||||
path = path.split(' ')[0]
|
||||
# "<path>?<params>" -> "<path>"
|
||||
path = path.split('?')[0]
|
||||
# "<path>#<fragment>" -> "<path>"
|
||||
path = path.split('#')[0]
|
||||
# "chrome://reftest/content/<path>" -> "layout/reftests/<path>"
|
||||
path = path.replace('chrome://reftest/content', 'layout/reftests')
|
||||
return path
|
||||
|
||||
def path_mod_jsreftest(self, path):
|
||||
# "<path-to>/jsreftest.html?test=<path>" -> "<path>"
|
||||
path = path.split('=')[1]
|
||||
# "<path>;assert" -> "<path>"
|
||||
path = path.split(';')[0]
|
||||
# "<rel-path>" -> "js/src/tests/<path>"
|
||||
path = os.path.join('js', 'src', 'tests', path)
|
||||
return path
|
||||
|
||||
def path_mod_marionette(self, path):
|
||||
# "<path> <test-name>" -> "<path>"
|
||||
path = path.split(' ')[0]
|
||||
# "part1\part2" -> "part1/part2"
|
||||
path = path.replace('\\', os.path.sep)
|
||||
return path
|
||||
|
||||
def path_mod_crashtest(self, path):
|
||||
# TODO: revisit the need for these transforms after bug 1596567
|
||||
import re
|
||||
# "chrome://reftest/content/crashtests/<path>" -> "<path>"
|
||||
path = path.replace('chrome://reftest/content/crashtests/', '')
|
||||
# "file://<path2>/reftests/tests/<path>" -> "<path>"
|
||||
path = re.sub(r'file://.*/reftest/tests/', '', path)
|
||||
# "http://<ip>/tests/<path>" -> "<path>"
|
||||
path = re.sub(r'http://\d*\.\d*\.\d*\.\d*:\d*/tests/', '', path)
|
||||
return path
|
||||
|
||||
def path_mod_wpt(self, path):
|
||||
if path[0] == os.path.sep:
|
||||
# "/<path>" -> "<path>"
|
||||
path = path[1:]
|
||||
# "<path>" -> "testing/web-platform/tests/<path>"
|
||||
path = os.path.join('testing', 'web-platform', 'tests', path)
|
||||
# "<path>?<params>" -> "<path>"
|
||||
path = path.split('?')[0]
|
||||
return path
|
||||
|
||||
def path_mod_jittest(self, path):
|
||||
# "part1\part2" -> "part1/part2"
|
||||
path = path.replace('\\', os.path.sep)
|
||||
# "<path>" -> "js/src/jit-test/tests/<path>"
|
||||
return os.path.join('js', 'src', 'jit-test', 'tests', path)
|
||||
|
||||
def add_activedata(self, branches, days, by_component):
|
||||
suites = {
|
||||
"reftest": self.path_mod_reftest,
|
||||
"web-platform-tests": self.path_mod_wpt,
|
||||
"web-platform-tests-reftests": self.path_mod_wpt,
|
||||
"jsreftest": self.path_mod_jsreftest,
|
||||
"crashtest": self.path_mod_crashtest,
|
||||
"xpcshell": None,
|
||||
"mochitest-plain": None,
|
||||
"jittest": self.path_mod_jittest,
|
||||
"mochitest-browser-chrome": None,
|
||||
"mochitest-media": None,
|
||||
"mochitest-devtools-chrome": None,
|
||||
"marionette": self.path_mod_marionette,
|
||||
"mochitest-chrome": None,
|
||||
"web-platform-tests-wdspec": self.path_mod_wpt,
|
||||
"geckoview-junit": None,
|
||||
"cppunittest": None,
|
||||
}
|
||||
for suite in suites:
|
||||
print("Adding ActiveData data for %s..." % suite)
|
||||
suite_clause = {"eq": {"run.suite.name": suite}}
|
||||
self.add_activedata_for_suite(branches, days, by_component,
|
||||
suite_clause, suites[suite])
|
||||
# remainder
|
||||
suite_clause = {"not": {"in": {"run.suite.name": list(suites)}}}
|
||||
self.add_activedata_for_suite(branches, days, by_component,
|
||||
suite_clause, None)
|
||||
ti = testinfo.TestInfoReport(verbose)
|
||||
ti.report(components, flavor, subsuite, paths,
|
||||
show_manifests, show_tests, show_summary, show_annotations,
|
||||
show_activedata,
|
||||
filter_values, filter_keys, show_components, output_file,
|
||||
branches, days)
|
||||
|
||||
|
||||
@CommandProvider
|
||||
|
||||
863
testing/testinfo.py
Normal file
863
testing/testinfo.py
Normal file
@@ -0,0 +1,863 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import, print_function
|
||||
|
||||
import datetime
|
||||
import errno
|
||||
import json
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import requests
|
||||
import subprocess
|
||||
import traceback
|
||||
import mozpack.path as mozpath
|
||||
from moztest.resolve import TestResolver, TestManifestLoader
|
||||
from mozfile import which
|
||||
|
||||
from mozbuild.base import MozbuildObject, MachCommandConditions as conditions
|
||||
|
||||
ACTIVEDATA_RECORD_LIMIT = 10000
|
||||
|
||||
|
||||
class TestInfo(object):
|
||||
"""
|
||||
Support 'mach test-info'.
|
||||
"""
|
||||
def __init__(self, verbose):
|
||||
self.verbose = verbose
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
self.build_obj = MozbuildObject.from_environment(cwd=here)
|
||||
|
||||
def log_verbose(self, what):
|
||||
if self.verbose:
|
||||
print(what)
|
||||
|
||||
def activedata_query(self, query):
|
||||
self.log_verbose(datetime.datetime.now())
|
||||
self.log_verbose(json.dumps(query))
|
||||
response = requests.post("http://activedata.allizom.org/query",
|
||||
data=json.dumps(query),
|
||||
stream=True)
|
||||
self.log_verbose(datetime.datetime.now())
|
||||
self.log_verbose(response)
|
||||
response.raise_for_status()
|
||||
data = response.json()["data"]
|
||||
self.log_verbose("response length: %d" % len(data))
|
||||
return data
|
||||
|
||||
|
||||
class TestInfoTests(TestInfo):
|
||||
"""
|
||||
Support 'mach test-info tests': Detailed report of specified tests.
|
||||
"""
|
||||
def __init__(self, verbose):
|
||||
TestInfo.__init__(self, verbose)
|
||||
|
||||
self._hg = None
|
||||
if conditions.is_hg(self.build_obj):
|
||||
self._hg = which('hg')
|
||||
if not self._hg:
|
||||
raise OSError(errno.ENOENT, "Could not find 'hg' on PATH.")
|
||||
|
||||
self._git = None
|
||||
if conditions.is_git(self.build_obj):
|
||||
self._git = which('git')
|
||||
if not self._git:
|
||||
raise OSError(errno.ENOENT, "Could not find 'git' on PATH.")
|
||||
|
||||
def find_in_hg_or_git(self, test_name):
|
||||
if self._hg:
|
||||
cmd = [self._hg, 'files', '-I', test_name]
|
||||
elif self._git:
|
||||
cmd = [self._git, 'ls-files', test_name]
|
||||
else:
|
||||
return None
|
||||
try:
|
||||
out = subprocess.check_output(cmd).splitlines()
|
||||
except subprocess.CalledProcessError:
|
||||
out = None
|
||||
return out
|
||||
|
||||
def set_test_name(self):
|
||||
# Generating a unified report for a specific test is complicated
|
||||
# by differences in the test name used in various data sources.
|
||||
# Consider:
|
||||
# - It is often convenient to request a report based only on
|
||||
# a short file name, rather than the full path;
|
||||
# - Bugs may be filed in bugzilla against a simple, short test
|
||||
# name or the full path to the test;
|
||||
# - In ActiveData, the full path is usually used, but sometimes
|
||||
# also includes additional path components outside of the
|
||||
# mercurial repo (common for reftests).
|
||||
# This function attempts to find appropriate names for different
|
||||
# queries based on the specified test name.
|
||||
|
||||
# full_test_name is full path to file in hg (or git)
|
||||
self.full_test_name = None
|
||||
out = self.find_in_hg_or_git(self.test_name)
|
||||
if out and len(out) == 1:
|
||||
self.full_test_name = out[0]
|
||||
elif out and len(out) > 1:
|
||||
print("Ambiguous test name specified. Found:")
|
||||
for line in out:
|
||||
print(line)
|
||||
else:
|
||||
out = self.find_in_hg_or_git('**/%s*' % self.test_name)
|
||||
if out and len(out) == 1:
|
||||
self.full_test_name = out[0]
|
||||
elif out and len(out) > 1:
|
||||
print("Ambiguous test name. Found:")
|
||||
for line in out:
|
||||
print(line)
|
||||
if self.full_test_name:
|
||||
self.full_test_name.replace(os.sep, posixpath.sep)
|
||||
print("Found %s in source control." % self.full_test_name)
|
||||
else:
|
||||
print("Unable to validate test name '%s'!" % self.test_name)
|
||||
self.full_test_name = self.test_name
|
||||
|
||||
# search for full_test_name in test manifests
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader)
|
||||
relpath = self.build_obj._wrap_path_argument(self.full_test_name).relpath()
|
||||
tests = list(resolver.resolve_tests(paths=[relpath]))
|
||||
if len(tests) == 1:
|
||||
relpath = self.build_obj._wrap_path_argument(tests[0]['manifest']).relpath()
|
||||
print("%s found in manifest %s" % (self.full_test_name, relpath))
|
||||
if tests[0].get('flavor'):
|
||||
print(" flavor: %s" % tests[0]['flavor'])
|
||||
if tests[0].get('skip-if'):
|
||||
print(" skip-if: %s" % tests[0]['skip-if'])
|
||||
if tests[0].get('fail-if'):
|
||||
print(" fail-if: %s" % tests[0]['fail-if'])
|
||||
elif len(tests) == 0:
|
||||
print("%s not found in any test manifest!" % self.full_test_name)
|
||||
else:
|
||||
print("%s found in more than one manifest!" % self.full_test_name)
|
||||
|
||||
# short_name is full_test_name without path
|
||||
self.short_name = None
|
||||
name_idx = self.full_test_name.rfind('/')
|
||||
if name_idx > 0:
|
||||
self.short_name = self.full_test_name[name_idx + 1:]
|
||||
if self.short_name and self.short_name == self.test_name:
|
||||
self.short_name = None
|
||||
|
||||
if not (self.show_results or self.show_durations or self.show_tasks):
|
||||
# no need to determine ActiveData name if not querying
|
||||
return
|
||||
|
||||
def set_activedata_test_name(self):
|
||||
# activedata_test_name is name in ActiveData
|
||||
self.activedata_test_name = None
|
||||
simple_names = [
|
||||
self.full_test_name,
|
||||
self.test_name,
|
||||
self.short_name
|
||||
]
|
||||
simple_names = [x for x in simple_names if x]
|
||||
searches = [
|
||||
{"in": {"result.test": simple_names}},
|
||||
]
|
||||
regex_names = [".*%s.*" % re.escape(x) for x in simple_names if x]
|
||||
for r in regex_names:
|
||||
searches.append({"regexp": {"result.test": r}})
|
||||
query = {
|
||||
"from": "unittest",
|
||||
"format": "list",
|
||||
"limit": 10,
|
||||
"groupby": ["result.test"],
|
||||
"where": {"and": [
|
||||
{"or": searches},
|
||||
{"in": {"build.branch": self.branches.split(',')}},
|
||||
{"gt": {"run.timestamp": {"date": self.start}}},
|
||||
{"lt": {"run.timestamp": {"date": self.end}}}
|
||||
]}
|
||||
}
|
||||
print("Querying ActiveData...") # Following query can take a long time
|
||||
data = self.activedata_query(query)
|
||||
if data and len(data) > 0:
|
||||
self.activedata_test_name = [
|
||||
d['result']['test']
|
||||
for p in simple_names + regex_names
|
||||
for d in data
|
||||
if re.match(p + "$", d['result']['test'])
|
||||
][0] # first match is best match
|
||||
if self.activedata_test_name:
|
||||
print("Found records matching '%s' in ActiveData." %
|
||||
self.activedata_test_name)
|
||||
else:
|
||||
print("Unable to find matching records in ActiveData; using %s!" %
|
||||
self.test_name)
|
||||
self.activedata_test_name = self.test_name
|
||||
|
||||
def get_run_types(self, record):
|
||||
types_label = ""
|
||||
if 'run' in record and 'type' in record['run']:
|
||||
run_types = record['run']['type']
|
||||
run_types = run_types if isinstance(run_types, list) else [run_types]
|
||||
fission = True if 'fis' in run_types else False
|
||||
for run_type in run_types:
|
||||
# chunked is not interesting
|
||||
if run_type == 'chunked':
|
||||
continue
|
||||
# fission implies e10s
|
||||
if fission and run_type == 'e10s':
|
||||
continue
|
||||
types_label += "-" + run_type
|
||||
return types_label
|
||||
|
||||
def get_platform(self, record):
|
||||
if 'platform' in record['build']:
|
||||
platform = record['build']['platform']
|
||||
else:
|
||||
platform = "-"
|
||||
tp = record['build']['type']
|
||||
if type(tp) is list:
|
||||
tp = "-".join(tp)
|
||||
return "%s/%s%s:" % (platform, tp, self.get_run_types(record))
|
||||
|
||||
def report_test_results(self):
|
||||
# Report test pass/fail summary from ActiveData
|
||||
query = {
|
||||
"from": "unittest",
|
||||
"format": "list",
|
||||
"limit": 100,
|
||||
"groupby": ["build.platform", "build.type", "run.type"],
|
||||
"select": [
|
||||
{"aggregate": "count"},
|
||||
{
|
||||
"name": "failures",
|
||||
"value": {"case": [
|
||||
{"when": {"eq": {"result.ok": "F"}}, "then": 1}
|
||||
]},
|
||||
"aggregate": "sum",
|
||||
"default": 0
|
||||
},
|
||||
{
|
||||
"name": "skips",
|
||||
"value": {"case": [
|
||||
{"when": {"eq": {"result.status": "SKIP"}}, "then": 1}
|
||||
]},
|
||||
"aggregate": "sum",
|
||||
"default": 0
|
||||
}
|
||||
],
|
||||
"where": {"and": [
|
||||
{"eq": {"result.test": self.activedata_test_name}},
|
||||
{"in": {"build.branch": self.branches.split(',')}},
|
||||
{"gt": {"run.timestamp": {"date": self.start}}},
|
||||
{"lt": {"run.timestamp": {"date": self.end}}}
|
||||
]}
|
||||
}
|
||||
print("\nTest results for %s on %s between %s and %s" %
|
||||
(self.activedata_test_name, self.branches, self.start, self.end))
|
||||
data = self.activedata_query(query)
|
||||
if data and len(data) > 0:
|
||||
data.sort(key=self.get_platform)
|
||||
worst_rate = 0.0
|
||||
worst_platform = None
|
||||
total_runs = 0
|
||||
total_failures = 0
|
||||
for record in data:
|
||||
platform = self.get_platform(record)
|
||||
if platform.startswith("-"):
|
||||
continue
|
||||
runs = record['count']
|
||||
total_runs = total_runs + runs
|
||||
failures = record.get('failures', 0)
|
||||
skips = record.get('skips', 0)
|
||||
total_failures = total_failures + failures
|
||||
rate = (float)(failures) / runs
|
||||
if rate >= worst_rate:
|
||||
worst_rate = rate
|
||||
worst_platform = platform
|
||||
worst_failures = failures
|
||||
worst_runs = runs
|
||||
print("%-40s %6d failures (%6d skipped) in %6d runs" % (
|
||||
platform, failures, skips, runs))
|
||||
print("\nTotal: %d failures in %d runs or %.3f failures/run" %
|
||||
(total_failures, total_runs, (float)(total_failures) / total_runs))
|
||||
if worst_failures > 0:
|
||||
print("Worst rate on %s %d failures in %d runs or %.3f failures/run" %
|
||||
(worst_platform, worst_failures, worst_runs, worst_rate))
|
||||
else:
|
||||
print("No test result data found.")
|
||||
|
||||
def report_test_durations(self):
|
||||
# Report test durations summary from ActiveData
|
||||
query = {
|
||||
"from": "unittest",
|
||||
"format": "list",
|
||||
"limit": 100,
|
||||
"groupby": ["build.platform", "build.type", "run.type"],
|
||||
"select": [
|
||||
{"value": "result.duration",
|
||||
"aggregate": "average", "name": "average"},
|
||||
{"value": "result.duration", "aggregate": "min", "name": "min"},
|
||||
{"value": "result.duration", "aggregate": "max", "name": "max"},
|
||||
{"aggregate": "count"}
|
||||
],
|
||||
"where": {"and": [
|
||||
{"eq": {"result.ok": "T"}},
|
||||
{"eq": {"result.test": self.activedata_test_name}},
|
||||
{"in": {"build.branch": self.branches.split(',')}},
|
||||
{"gt": {"run.timestamp": {"date": self.start}}},
|
||||
{"lt": {"run.timestamp": {"date": self.end}}}
|
||||
]}
|
||||
}
|
||||
data = self.activedata_query(query)
|
||||
print("\nTest durations for %s on %s between %s and %s" %
|
||||
(self.activedata_test_name, self.branches, self.start, self.end))
|
||||
if data and len(data) > 0:
|
||||
data.sort(key=self.get_platform)
|
||||
for record in data:
|
||||
platform = self.get_platform(record)
|
||||
if platform.startswith("-"):
|
||||
continue
|
||||
print("%-40s %6.2f s (%.2f s - %.2f s over %d runs)" % (
|
||||
platform, record['average'], record['min'],
|
||||
record['max'], record['count']))
|
||||
else:
|
||||
print("No test durations found.")
|
||||
|
||||
def report_test_tasks(self):
|
||||
# Report test tasks summary from ActiveData
|
||||
query = {
|
||||
"from": "unittest",
|
||||
"format": "list",
|
||||
"limit": 1000,
|
||||
"select": ["build.platform", "build.type", "run.type", "run.name"],
|
||||
"where": {"and": [
|
||||
{"eq": {"result.test": self.activedata_test_name}},
|
||||
{"in": {"build.branch": self.branches.split(',')}},
|
||||
{"gt": {"run.timestamp": {"date": self.start}}},
|
||||
{"lt": {"run.timestamp": {"date": self.end}}}
|
||||
]}
|
||||
}
|
||||
data = self.activedata_query(query)
|
||||
print("\nTest tasks for %s on %s between %s and %s" %
|
||||
(self.activedata_test_name, self.branches, self.start, self.end))
|
||||
if data and len(data) > 0:
|
||||
data.sort(key=self.get_platform)
|
||||
consolidated = {}
|
||||
for record in data:
|
||||
platform = self.get_platform(record)
|
||||
if platform not in consolidated:
|
||||
consolidated[platform] = {}
|
||||
if record['run']['name'] in consolidated[platform]:
|
||||
consolidated[platform][record['run']['name']] += 1
|
||||
else:
|
||||
consolidated[platform][record['run']['name']] = 1
|
||||
for key in sorted(consolidated.keys()):
|
||||
tasks = ""
|
||||
for task in consolidated[key].keys():
|
||||
if tasks:
|
||||
tasks += "\n%-40s " % ""
|
||||
tasks += task
|
||||
tasks += " in %d runs" % consolidated[key][task]
|
||||
print("%-40s %s" % (key, tasks))
|
||||
else:
|
||||
print("No test tasks found.")
|
||||
|
||||
def report_bugs(self):
|
||||
# Report open bugs matching test name
|
||||
search = self.full_test_name
|
||||
if self.test_name:
|
||||
search = '%s,%s' % (search, self.test_name)
|
||||
if self.short_name:
|
||||
search = '%s,%s' % (search, self.short_name)
|
||||
payload = {'quicksearch': search,
|
||||
'include_fields': 'id,summary'}
|
||||
response = requests.get('https://bugzilla.mozilla.org/rest/bug',
|
||||
payload)
|
||||
response.raise_for_status()
|
||||
json_response = response.json()
|
||||
print("\nBugzilla quick search for '%s':" % search)
|
||||
if 'bugs' in json_response:
|
||||
for bug in json_response['bugs']:
|
||||
print("Bug %s: %s" % (bug['id'], bug['summary']))
|
||||
else:
|
||||
print("No bugs found.")
|
||||
|
||||
def report(self, test_names, branches, start, end,
|
||||
show_info, show_results, show_durations, show_tasks, show_bugs):
|
||||
self.branches = branches
|
||||
self.start = start
|
||||
self.end = end
|
||||
self.show_info = show_info
|
||||
self.show_results = show_results
|
||||
self.show_durations = show_durations
|
||||
self.show_tasks = show_tasks
|
||||
|
||||
if (not self.show_info and
|
||||
not self.show_results and
|
||||
not self.show_durations and
|
||||
not self.show_tasks and
|
||||
not show_bugs):
|
||||
# by default, show everything
|
||||
self.show_info = True
|
||||
self.show_results = True
|
||||
self.show_durations = True
|
||||
self.show_tasks = True
|
||||
show_bugs = True
|
||||
|
||||
for test_name in test_names:
|
||||
print("===== %s =====" % test_name)
|
||||
self.test_name = test_name
|
||||
if len(self.test_name) < 6:
|
||||
print("'%s' is too short for a test name!" % self.test_name)
|
||||
continue
|
||||
self.set_test_name()
|
||||
if show_bugs:
|
||||
self.report_bugs()
|
||||
self.set_activedata_test_name()
|
||||
if self.show_results:
|
||||
self.report_test_results()
|
||||
if self.show_durations:
|
||||
self.report_test_durations()
|
||||
if self.show_tasks:
|
||||
self.report_test_tasks()
|
||||
|
||||
|
||||
class TestInfoLongRunningTasks(TestInfo):
|
||||
"""
|
||||
Support 'mach test-info long-tasks': Summary of tasks approaching their max-run-time.
|
||||
"""
|
||||
def __init__(self, verbose):
|
||||
TestInfo.__init__(self, verbose)
|
||||
|
||||
def report(self, branches, start, end, threshold_pct, filter_threshold_pct):
|
||||
|
||||
def get_long_running_ratio(record):
|
||||
count = record['count']
|
||||
tasks_gt_pct = record['tasks_gt_pct']
|
||||
return count / tasks_gt_pct
|
||||
|
||||
# Search test durations in ActiveData for long-running tests
|
||||
query = {
|
||||
"from": "task",
|
||||
"format": "list",
|
||||
"groupby": ["run.name"],
|
||||
"limit": 1000,
|
||||
"select": [
|
||||
{
|
||||
"value": "task.maxRunTime",
|
||||
"aggregate": "median",
|
||||
"name": "max_run_time"
|
||||
},
|
||||
{
|
||||
"aggregate": "count"
|
||||
},
|
||||
{
|
||||
"value": {
|
||||
"when": {
|
||||
"gt": [
|
||||
{
|
||||
"div": ["action.duration", "task.maxRunTime"]
|
||||
}, threshold_pct/100.0
|
||||
]
|
||||
},
|
||||
"then": 1
|
||||
},
|
||||
"aggregate": "sum",
|
||||
"name": "tasks_gt_pct"
|
||||
},
|
||||
],
|
||||
"where": {"and": [
|
||||
{"in": {"build.branch": branches.split(',')}},
|
||||
{"gt": {"task.run.start_time": {"date": start}}},
|
||||
{"lte": {"task.run.start_time": {"date": end}}},
|
||||
{"eq": {"task.state": "completed"}},
|
||||
]}
|
||||
}
|
||||
data = self.activedata_query(query)
|
||||
print("\nTasks nearing their max-run-time on %s between %s and %s" %
|
||||
(branches, start, end))
|
||||
if data and len(data) > 0:
|
||||
filtered = []
|
||||
for record in data:
|
||||
if 'tasks_gt_pct' in record:
|
||||
count = record['count']
|
||||
tasks_gt_pct = record['tasks_gt_pct']
|
||||
if float(tasks_gt_pct) / count > filter_threshold_pct / 100.0:
|
||||
filtered.append(record)
|
||||
filtered.sort(key=get_long_running_ratio)
|
||||
if not filtered:
|
||||
print("No long running tasks found.")
|
||||
for record in filtered:
|
||||
name = record['run']['name']
|
||||
count = record['count']
|
||||
max_run_time = record['max_run_time']
|
||||
tasks_gt_pct = record['tasks_gt_pct']
|
||||
print("%-55s: %d of %d runs (%.1f%%) exceeded %d%% of max-run-time (%d s)" %
|
||||
(name, tasks_gt_pct, count, tasks_gt_pct * 100 / count,
|
||||
threshold_pct, max_run_time))
|
||||
else:
|
||||
print("No tasks found.")
|
||||
|
||||
|
||||
class TestInfoReport(TestInfo):
|
||||
"""
|
||||
Support 'mach test-info report': Report of test runs summarized by
|
||||
manifest and component.
|
||||
"""
|
||||
def __init__(self, verbose):
|
||||
TestInfo.__init__(self, verbose)
|
||||
|
||||
def add_activedata_for_suite(self, branches, days, by_component,
|
||||
suite_clause, path_mod):
|
||||
dates_clause = {"date": "today-%dday" % days}
|
||||
ad_query = {
|
||||
"from": "unittest",
|
||||
"limit": ACTIVEDATA_RECORD_LIMIT,
|
||||
"format": "list",
|
||||
"groupby": ["result.test"],
|
||||
"select": [
|
||||
{
|
||||
"name": "result.count",
|
||||
"aggregate": "count"
|
||||
},
|
||||
{
|
||||
"name": "result.duration",
|
||||
"value": "result.duration",
|
||||
"aggregate": "sum"
|
||||
},
|
||||
{
|
||||
"name": "result.failures",
|
||||
"value": {"case": [
|
||||
{"when": {"eq": {"result.ok": "F"}}, "then": 1}
|
||||
]},
|
||||
"aggregate": "sum",
|
||||
"default": 0
|
||||
},
|
||||
{
|
||||
"name": "result.skips",
|
||||
"value": {"case": [
|
||||
{"when": {"eq": {"result.status": "SKIP"}}, "then": 1}
|
||||
]},
|
||||
"aggregate": "sum",
|
||||
"default": 0
|
||||
}
|
||||
],
|
||||
"where": {"and": [
|
||||
suite_clause,
|
||||
{"in": {"repo.branch.name": branches.split(',')}},
|
||||
{"gt": {"run.timestamp": dates_clause}}
|
||||
]}
|
||||
}
|
||||
ad_response = self.activedata_query(ad_query)
|
||||
if len(ad_response) == ACTIVEDATA_RECORD_LIMIT:
|
||||
print("warning: limit reached; some data may be missing")
|
||||
for record in ad_response:
|
||||
if 'result' in record:
|
||||
result = record['result']
|
||||
self.update_report(by_component, result, path_mod)
|
||||
|
||||
def update_report(self, by_component, result, path_mod):
|
||||
if 'test' in result and 'tests' in by_component:
|
||||
test = result['test']
|
||||
if path_mod:
|
||||
test = path_mod(test)
|
||||
for bc in by_component['tests']:
|
||||
for item in by_component['tests'][bc]:
|
||||
if test == item['test']:
|
||||
seconds = result.get('duration', 'unknown')
|
||||
if seconds != 'unknown':
|
||||
seconds = round(seconds, 2)
|
||||
item['total run time, seconds'] = seconds
|
||||
item['total runs'] = result.get('count', 'unknown')
|
||||
item['skipped runs'] = result.get('skips', 'unknown')
|
||||
item['failed runs'] = result.get('failures', 'unknown')
|
||||
return True
|
||||
return False
|
||||
|
||||
def path_mod_reftest(self, path):
|
||||
# "<path1> == <path2>" -> "<path1>"
|
||||
path = path.split(' ')[0]
|
||||
# "<path>?<params>" -> "<path>"
|
||||
path = path.split('?')[0]
|
||||
# "<path>#<fragment>" -> "<path>"
|
||||
path = path.split('#')[0]
|
||||
# "chrome://reftest/content/<path>" -> "layout/reftests/<path>"
|
||||
path = path.replace('chrome://reftest/content', 'layout/reftests')
|
||||
return path
|
||||
|
||||
def path_mod_jsreftest(self, path):
|
||||
# "<path-to>/jsreftest.html?test=<path>" -> "<path>"
|
||||
path = path.split('=')[1]
|
||||
# "<path>;assert" -> "<path>"
|
||||
path = path.split(';')[0]
|
||||
# "<rel-path>" -> "js/src/tests/<path>"
|
||||
path = os.path.join('js', 'src', 'tests', path)
|
||||
return path
|
||||
|
||||
def path_mod_marionette(self, path):
|
||||
# "<path> <test-name>" -> "<path>"
|
||||
path = path.split(' ')[0]
|
||||
# "part1\part2" -> "part1/part2"
|
||||
path = path.replace('\\', os.path.sep)
|
||||
return path
|
||||
|
||||
def path_mod_crashtest(self, path):
|
||||
# TODO: revisit the need for these transforms after bug 1596567
|
||||
# "chrome://reftest/content/crashtests/<path>" -> "<path>"
|
||||
path = path.replace('chrome://reftest/content/crashtests/', '')
|
||||
# "file://<path2>/reftests/tests/<path>" -> "<path>"
|
||||
path = re.sub(r'file://.*/reftest/tests/', '', path)
|
||||
# "http://<ip>/tests/<path>" -> "<path>"
|
||||
path = re.sub(r'http://\d*\.\d*\.\d*\.\d*:\d*/tests/', '', path)
|
||||
return path
|
||||
|
||||
def path_mod_wpt(self, path):
|
||||
if path[0] == os.path.sep:
|
||||
# "/<path>" -> "<path>"
|
||||
path = path[1:]
|
||||
# "<path>" -> "testing/web-platform/tests/<path>"
|
||||
path = os.path.join('testing', 'web-platform', 'tests', path)
|
||||
# "<path>?<params>" -> "<path>"
|
||||
path = path.split('?')[0]
|
||||
return path
|
||||
|
||||
def path_mod_jittest(self, path):
|
||||
# "part1\part2" -> "part1/part2"
|
||||
path = path.replace('\\', os.path.sep)
|
||||
# "<path>" -> "js/src/jit-test/tests/<path>"
|
||||
return os.path.join('js', 'src', 'jit-test', 'tests', path)
|
||||
|
||||
def add_activedata(self, branches, days, by_component):
|
||||
suites = {
|
||||
"reftest": self.path_mod_reftest,
|
||||
"web-platform-tests": self.path_mod_wpt,
|
||||
"web-platform-tests-reftests": self.path_mod_wpt,
|
||||
"jsreftest": self.path_mod_jsreftest,
|
||||
"crashtest": self.path_mod_crashtest,
|
||||
"xpcshell": None,
|
||||
"mochitest-plain": None,
|
||||
"jittest": self.path_mod_jittest,
|
||||
"mochitest-browser-chrome": None,
|
||||
"mochitest-media": None,
|
||||
"mochitest-devtools-chrome": None,
|
||||
"marionette": self.path_mod_marionette,
|
||||
"mochitest-chrome": None,
|
||||
"web-platform-tests-wdspec": self.path_mod_wpt,
|
||||
"geckoview-junit": None,
|
||||
"cppunittest": None,
|
||||
}
|
||||
for suite in suites:
|
||||
print("Adding ActiveData data for %s..." % suite)
|
||||
suite_clause = {"eq": {"run.suite.name": suite}}
|
||||
self.add_activedata_for_suite(branches, days, by_component,
|
||||
suite_clause, suites[suite])
|
||||
# remainder
|
||||
suite_clause = {"not": {"in": {"run.suite.name": list(suites)}}}
|
||||
self.add_activedata_for_suite(branches, days, by_component,
|
||||
suite_clause, None)
|
||||
|
||||
def report(self, components, flavor, subsuite, paths,
|
||||
show_manifests, show_tests, show_summary, show_annotations,
|
||||
show_activedata,
|
||||
filter_values, filter_keys, show_components, output_file,
|
||||
branches, days):
|
||||
|
||||
def matches_filters(test):
|
||||
'''
|
||||
Return True if all of the requested filter_values are found in this test;
|
||||
if filter_keys are specified, restrict search to those test keys.
|
||||
'''
|
||||
for value in filter_values:
|
||||
value_found = False
|
||||
for key in test:
|
||||
if not filter_keys or key in filter_keys:
|
||||
if re.search(value, test[key]):
|
||||
value_found = True
|
||||
break
|
||||
if not value_found:
|
||||
return False
|
||||
return True
|
||||
|
||||
# Ensure useful report by default
|
||||
if not show_manifests and not show_tests and not show_summary and not show_annotations:
|
||||
show_manifests = True
|
||||
show_summary = True
|
||||
|
||||
by_component = {}
|
||||
if components:
|
||||
components = components.split(',')
|
||||
if filter_keys:
|
||||
filter_keys = filter_keys.split(',')
|
||||
if filter_values:
|
||||
filter_values = filter_values.split(',')
|
||||
else:
|
||||
filter_values = []
|
||||
display_keys = (filter_keys or []) + ['skip-if', 'fail-if', 'fails-if']
|
||||
display_keys = set(display_keys)
|
||||
|
||||
print("Finding tests...")
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
resolver = TestResolver.from_environment(cwd=here, loader_cls=TestManifestLoader)
|
||||
tests = list(resolver.resolve_tests(paths=paths, flavor=flavor,
|
||||
subsuite=subsuite))
|
||||
|
||||
manifest_paths = set()
|
||||
for t in tests:
|
||||
if 'manifest' in t and t['manifest'] is not None:
|
||||
manifest_paths.add(t['manifest'])
|
||||
manifest_count = len(manifest_paths)
|
||||
print("Resolver found {} tests, {} manifests".format(len(tests), manifest_count))
|
||||
|
||||
if show_manifests:
|
||||
topsrcdir = self.build_obj.topsrcdir
|
||||
by_component['manifests'] = {}
|
||||
manifest_paths = list(manifest_paths)
|
||||
manifest_paths.sort()
|
||||
relpaths = []
|
||||
for manifest_path in manifest_paths:
|
||||
relpath = mozpath.relpath(manifest_path, topsrcdir)
|
||||
if mozpath.commonprefix((manifest_path, topsrcdir)) != topsrcdir:
|
||||
continue
|
||||
relpaths.append(relpath)
|
||||
reader = self.build_obj.mozbuild_reader(config_mode='empty')
|
||||
files_info = reader.files_info(relpaths)
|
||||
for manifest_path in manifest_paths:
|
||||
relpath = mozpath.relpath(manifest_path, topsrcdir)
|
||||
if mozpath.commonprefix((manifest_path, topsrcdir)) != topsrcdir:
|
||||
continue
|
||||
manifest_info = None
|
||||
if relpath in files_info:
|
||||
bug_component = files_info[relpath].get('BUG_COMPONENT')
|
||||
key = "{}::{}".format(bug_component.product, bug_component.component)
|
||||
if (not components) or (key in components):
|
||||
manifest_info = {
|
||||
'manifest': relpath,
|
||||
'tests': 0,
|
||||
'skipped': 0
|
||||
}
|
||||
rkey = key if show_components else 'all'
|
||||
if rkey in by_component['manifests']:
|
||||
by_component['manifests'][rkey].append(manifest_info)
|
||||
else:
|
||||
by_component['manifests'][rkey] = [manifest_info]
|
||||
if manifest_info:
|
||||
for t in tests:
|
||||
if t['manifest'] == manifest_path:
|
||||
manifest_info['tests'] += 1
|
||||
if t.get('skip-if'):
|
||||
manifest_info['skipped'] += 1
|
||||
for key in by_component['manifests']:
|
||||
by_component['manifests'][key].sort()
|
||||
|
||||
if show_tests:
|
||||
by_component['tests'] = {}
|
||||
|
||||
if show_tests or show_summary or show_annotations:
|
||||
test_count = 0
|
||||
failed_count = 0
|
||||
skipped_count = 0
|
||||
annotation_count = 0
|
||||
condition_count = 0
|
||||
component_set = set()
|
||||
relpaths = []
|
||||
conditions = {}
|
||||
known_unconditional_annotations = ['skip', 'fail', 'asserts', 'random']
|
||||
known_conditional_annotations = ['skip-if', 'fail-if', 'run-if',
|
||||
'fails-if', 'fuzzy-if', 'random-if', 'asserts-if']
|
||||
for t in tests:
|
||||
relpath = t.get('srcdir_relpath')
|
||||
relpaths.append(relpath)
|
||||
reader = self.build_obj.mozbuild_reader(config_mode='empty')
|
||||
files_info = reader.files_info(relpaths)
|
||||
for t in tests:
|
||||
if not matches_filters(t):
|
||||
continue
|
||||
if 'referenced-test' in t:
|
||||
# Avoid double-counting reftests: disregard reference file entries
|
||||
continue
|
||||
if show_annotations:
|
||||
for key in t:
|
||||
if key in known_unconditional_annotations:
|
||||
annotation_count += 1
|
||||
if key in known_conditional_annotations:
|
||||
annotation_count += 1
|
||||
# Here 'key' is a manifest annotation type like 'skip-if' and t[key]
|
||||
# is the associated condition. For example, the manifestparser
|
||||
# manifest annotation, "skip-if = os == 'win'", is expected to be
|
||||
# encoded as t['skip-if'] = "os == 'win'".
|
||||
# To allow for reftest manifests, t[key] may have multiple entries
|
||||
# separated by ';', each corresponding to a condition for that test
|
||||
# and annotation type. For example,
|
||||
# "skip-if(Android&&webrender) skip-if(OSX)", would be
|
||||
# encoded as t['skip-if'] = "Android&&webrender;OSX".
|
||||
annotation_conditions = t[key].split(';')
|
||||
for condition in annotation_conditions:
|
||||
condition_count += 1
|
||||
# Trim reftest fuzzy-if ranges: everything after the first comma
|
||||
# eg. "Android,0-2,1-3" -> "Android"
|
||||
condition = condition.split(',')[0]
|
||||
if condition not in conditions:
|
||||
conditions[condition] = 0
|
||||
conditions[condition] += 1
|
||||
test_count += 1
|
||||
relpath = t.get('srcdir_relpath')
|
||||
if relpath in files_info:
|
||||
bug_component = files_info[relpath].get('BUG_COMPONENT')
|
||||
key = "{}::{}".format(bug_component.product, bug_component.component)
|
||||
if (not components) or (key in components):
|
||||
component_set.add(key)
|
||||
test_info = {'test': relpath}
|
||||
for test_key in display_keys:
|
||||
value = t.get(test_key)
|
||||
if value:
|
||||
test_info[test_key] = value
|
||||
if t.get('fail-if'):
|
||||
failed_count += 1
|
||||
if t.get('fails-if'):
|
||||
failed_count += 1
|
||||
if t.get('skip-if'):
|
||||
skipped_count += 1
|
||||
if show_tests:
|
||||
rkey = key if show_components else 'all'
|
||||
if rkey in by_component['tests']:
|
||||
by_component['tests'][rkey].append(test_info)
|
||||
else:
|
||||
by_component['tests'][rkey] = [test_info]
|
||||
if show_tests:
|
||||
for key in by_component['tests']:
|
||||
by_component['tests'][key].sort(key=lambda k: k['test'])
|
||||
|
||||
if show_activedata:
|
||||
try:
|
||||
self.add_activedata(branches, days, by_component)
|
||||
except Exception:
|
||||
print("Failed to retrieve some ActiveData data.")
|
||||
traceback.print_exc()
|
||||
|
||||
if show_summary:
|
||||
by_component['summary'] = {}
|
||||
by_component['summary']['components'] = len(component_set)
|
||||
by_component['summary']['manifests'] = manifest_count
|
||||
by_component['summary']['tests'] = test_count
|
||||
by_component['summary']['failed tests'] = failed_count
|
||||
by_component['summary']['skipped tests'] = skipped_count
|
||||
|
||||
if show_annotations:
|
||||
by_component['annotations'] = {}
|
||||
by_component['annotations']['total annotations'] = annotation_count
|
||||
by_component['annotations']['total conditions'] = condition_count
|
||||
by_component['annotations']['unique conditions'] = len(conditions)
|
||||
by_component['annotations']['conditions'] = conditions
|
||||
|
||||
json_report = json.dumps(by_component, indent=2, sort_keys=True)
|
||||
if output_file:
|
||||
output_file = os.path.abspath(output_file)
|
||||
output_dir = os.path.dirname(output_file)
|
||||
if not os.path.isdir(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
|
||||
with open(output_file, 'w') as f:
|
||||
f.write(json_report)
|
||||
else:
|
||||
print(json_report)
|
||||
Reference in New Issue
Block a user