8eedbcea31
This patch adds the run methods for the perf selector as well as the entry point for the mach command. It also produces the Perfherder URL at the end. Depends on D160417 Differential Revision: https://phabricator.services.mozilla.com/D160418
694 lines
24 KiB
Python
694 lines
24 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
import os
|
|
import itertools
|
|
import re
|
|
import sys
|
|
|
|
from contextlib import redirect_stdout
|
|
|
|
from mozbuild.base import MozbuildObject
|
|
from mozversioncontrol import get_repository_object
|
|
|
|
from .compare import CompareParser
|
|
from ..push import push_to_try, generate_try_task_config
|
|
from ..util.fzf import (
|
|
build_base_cmd,
|
|
fzf_bootstrap,
|
|
FZF_NOT_FOUND,
|
|
setup_tasks_for_fzf,
|
|
run_fzf,
|
|
)
|
|
|
|
|
|
here = os.path.abspath(os.path.dirname(__file__))
|
|
build = MozbuildObject.from_environment(cwd=here)
|
|
|
|
PERFHERDER_BASE_URL = (
|
|
"https://treeherder.mozilla.org/perfherder/"
|
|
"compare?originalProject=try&originalRevision=%s&newProject=try&newRevision=%s"
|
|
)
|
|
|
|
# Prevent users from running more than 300 tests at once. It's possible, but
|
|
# it's more likely that a query is broken and is selecting far too much.
|
|
MAX_PERF_TASKS = 300
|
|
REVISION_MATCHER = re.compile(r"remote:.*/try/rev/([\w]*)[ \t]*$")
|
|
|
|
|
|
class LogProcessor:
|
|
def __init__(self):
|
|
self.buf = ""
|
|
self.stdout = sys.__stdout__
|
|
self._revision = None
|
|
|
|
@property
|
|
def revision(self):
|
|
return self._revision
|
|
|
|
def write(self, buf):
|
|
while buf:
|
|
try:
|
|
newline_index = buf.index("\n")
|
|
except ValueError:
|
|
# No newline, wait for next call
|
|
self.buf += buf
|
|
break
|
|
|
|
# Get data up to next newline and combine with previously buffered data
|
|
data = self.buf + buf[: newline_index + 1]
|
|
buf = buf[newline_index + 1 :]
|
|
|
|
# Reset buffer then output line
|
|
self.buf = ""
|
|
if data.strip() == "":
|
|
continue
|
|
self.stdout.write(data.strip("\n") + "\n")
|
|
|
|
# Check if a temporary commit wa created
|
|
match = REVISION_MATCHER.match(data)
|
|
if match:
|
|
# Last line found is the revision we want
|
|
self._revision = match.group(1)
|
|
|
|
|
|
class PerfParser(CompareParser):
|
|
name = "perf"
|
|
common_groups = ["push", "task"]
|
|
task_configs = [
|
|
"artifact",
|
|
"browsertime",
|
|
"disable-pgo",
|
|
"env",
|
|
"gecko-profile",
|
|
"path",
|
|
"rebuild",
|
|
]
|
|
|
|
platforms = {
|
|
"android-a51": {
|
|
"query": "'android 'a51 'shippable 'aarch64",
|
|
"platform": "android",
|
|
},
|
|
"android": {
|
|
# The android, and android-a51 queries are expected to be the same,
|
|
# we don't want to run the tests on other mobile platforms.
|
|
"query": "'android 'a51 'shippable 'aarch64",
|
|
"platform": "android",
|
|
},
|
|
"windows": {
|
|
"query": "!-32 'windows 'shippable",
|
|
"platform": "desktop",
|
|
},
|
|
"linux": {
|
|
"query": "!clang 'linux 'shippable",
|
|
"platform": "desktop",
|
|
},
|
|
"macosx": {
|
|
"query": "'osx 'shippable",
|
|
"platform": "desktop",
|
|
},
|
|
"desktop": {
|
|
"query": "!android 'shippable !-32 !clang",
|
|
"platform": "desktop",
|
|
},
|
|
}
|
|
|
|
apps = {
|
|
"firefox": {
|
|
"query": "!chrom !geckoview !fenix",
|
|
"platforms": ["desktop"],
|
|
},
|
|
"chrome": {
|
|
"query": "'chrome",
|
|
"platforms": ["desktop"],
|
|
},
|
|
"chromium": {
|
|
"query": "'chromium",
|
|
"platforms": ["desktop"],
|
|
},
|
|
"geckoview": {
|
|
"query": "'geckoview",
|
|
"platforms": ["android"],
|
|
},
|
|
"fenix": {
|
|
"query": "'fenix",
|
|
"platforms": ["android"],
|
|
},
|
|
"chrome-m": {
|
|
"query": "'chrome-m",
|
|
"platforms": ["android"],
|
|
},
|
|
}
|
|
|
|
variants = {
|
|
"no-fission": {
|
|
"query": "'nofis",
|
|
"negation": "!nofis",
|
|
"platforms": ["android"],
|
|
"apps": ["fenix", "geckoview"],
|
|
},
|
|
"bytecode-cached": {
|
|
"query": "'bytecode",
|
|
"negation": "!bytecode",
|
|
"platforms": ["desktop"],
|
|
"apps": ["firefox"],
|
|
},
|
|
"live-sites": {
|
|
"query": "'live",
|
|
"negation": "!live",
|
|
"platforms": ["desktop", "android"],
|
|
"apps": list(apps.keys()),
|
|
},
|
|
"profiling": {
|
|
"query": "'profil",
|
|
"negation": "!profil",
|
|
"platforms": ["desktop", "android"],
|
|
"apps": ["firefox", "geckoview", "fenix"],
|
|
},
|
|
}
|
|
|
|
# The tasks field can be used to hardcode tasks to run,
|
|
# otherwise we run the query selector combined with the platform
|
|
# and variants queries
|
|
categories = {
|
|
"Pageload": {
|
|
"query": "'browsertime 'tp6",
|
|
"tasks": [],
|
|
},
|
|
"Pageload (essential)": {
|
|
"query": "'browsertime 'tp6 'essential",
|
|
"tasks": [],
|
|
},
|
|
"Pageload (live)": {
|
|
"query": "'browsertime 'tp6 'live",
|
|
"tasks": [],
|
|
},
|
|
"Bytecode Cached": {
|
|
"query": "'browsertime 'bytecode",
|
|
"tasks": [],
|
|
},
|
|
"Responsiveness": {
|
|
"query": "'browsertime 'responsive",
|
|
"tasks": [],
|
|
},
|
|
"Benchmarks": {
|
|
"query": "'browsertime 'benchmark",
|
|
"tasks": [],
|
|
},
|
|
}
|
|
|
|
arguments = [
|
|
[
|
|
["--show-all"],
|
|
{
|
|
"action": "store_true",
|
|
"default": False,
|
|
"help": "Show all available tasks.",
|
|
},
|
|
],
|
|
[
|
|
["--android"],
|
|
{
|
|
"action": "store_true",
|
|
"default": False,
|
|
"help": "Show android test categories (disabled by default).",
|
|
},
|
|
],
|
|
[
|
|
["--chrome"],
|
|
{
|
|
"action": "store_true",
|
|
"default": False,
|
|
"help": "Show tests available for Chrome-based browsers "
|
|
"(disabled by default).",
|
|
},
|
|
],
|
|
[
|
|
["--live-sites"],
|
|
{
|
|
"action": "store_true",
|
|
"default": False,
|
|
"help": "Run tasks with live sites (if possible). "
|
|
"You can also use the `live-sites` variant.",
|
|
},
|
|
],
|
|
[
|
|
["--profile"],
|
|
{
|
|
"action": "store_true",
|
|
"default": False,
|
|
"help": "Run tasks with profiling (if possible). "
|
|
"You can also use the `profiling` variant.",
|
|
},
|
|
],
|
|
[
|
|
["--variants"],
|
|
{
|
|
"nargs": "*",
|
|
"type": str,
|
|
"default": [],
|
|
"dest": "requested_variants",
|
|
"choices": list(variants.keys()),
|
|
"help": "Show android test categories.",
|
|
},
|
|
],
|
|
[
|
|
["--platforms"],
|
|
{
|
|
"nargs": "*",
|
|
"type": str,
|
|
"default": [],
|
|
"dest": "requested_platforms",
|
|
"choices": list(platforms.keys()),
|
|
"help": "Select specific platforms to target. Android only "
|
|
"available with --android.",
|
|
},
|
|
],
|
|
[
|
|
["--apps"],
|
|
{
|
|
"nargs": "*",
|
|
"type": str,
|
|
"default": [],
|
|
"dest": "requested_apps",
|
|
"choices": list(apps.keys()),
|
|
"help": "Select specific applications to target.",
|
|
},
|
|
],
|
|
]
|
|
|
|
def get_tasks(base_cmd, queries, query_arg=None, candidate_tasks=None):
|
|
cmd = base_cmd[:]
|
|
if query_arg:
|
|
cmd.extend(["-f", query_arg])
|
|
|
|
query_str, tasks = run_fzf(cmd, sorted(candidate_tasks))
|
|
queries.append(query_str)
|
|
return set(tasks)
|
|
|
|
def get_perf_tasks(base_cmd, all_tg_tasks, perf_categories):
|
|
# Convert the categories to tasks
|
|
selected_tasks = set()
|
|
queries = []
|
|
|
|
selected_categories = PerfParser.get_tasks(
|
|
base_cmd, queries, None, perf_categories
|
|
)
|
|
|
|
for category, category_info in perf_categories.items():
|
|
if category not in selected_categories:
|
|
continue
|
|
print("Gathering tasks for %s category" % category)
|
|
|
|
# Either perform a query to get the tasks (recommended), or
|
|
# use a hardcoded task list
|
|
category_tasks = set()
|
|
if category_info["queries"]:
|
|
print("Executing queries: %s" % ", ".join(category_info["queries"]))
|
|
|
|
for perf_query in category_info["queries"]:
|
|
if not category_tasks:
|
|
# Get all tasks selected with the first query
|
|
category_tasks |= PerfParser.get_tasks(
|
|
base_cmd, queries, perf_query, all_tg_tasks
|
|
)
|
|
else:
|
|
# Keep only those tasks that matched in all previous queries
|
|
category_tasks &= PerfParser.get_tasks(
|
|
base_cmd, queries, perf_query, category_tasks
|
|
)
|
|
|
|
if len(category_tasks) == 0:
|
|
print("Failed to find any tasks for query: %s" % perf_query)
|
|
break
|
|
else:
|
|
category_tasks = set(category_info["tasks"]) & all_tg_tasks
|
|
if category_tasks != set(category_info["tasks"]):
|
|
print(
|
|
"Some expected tasks could not be found: %s"
|
|
% ", ".join(category_info["tasks"] - category_tasks)
|
|
)
|
|
|
|
if not category_tasks:
|
|
print("Could not find any tasks for category %s" % category)
|
|
else:
|
|
# Add the new tasks to the currently selected ones
|
|
selected_tasks |= category_tasks
|
|
|
|
if len(selected_tasks) > MAX_PERF_TASKS:
|
|
print(
|
|
"That's a lot of tests selected (%s)!\n"
|
|
"These tests won't be triggered. If this was unexpected, "
|
|
"please file a bug in Testing :: Performance." % MAX_PERF_TASKS
|
|
)
|
|
return [], [], []
|
|
|
|
return selected_tasks, selected_categories, queries
|
|
|
|
def expand_categories(
|
|
android=False,
|
|
chrome=False,
|
|
live_sites=False,
|
|
profile=False,
|
|
requested_variants=[],
|
|
requested_platforms=[],
|
|
requested_apps=[],
|
|
):
|
|
"""Setup the perf categories.
|
|
|
|
This has multiple steps:
|
|
(1) Expand the variants to all possible combinations
|
|
(2) Expand the test categories for all valid platform+app combinations
|
|
(3) Expand the categories from (2) into all possible combinations,
|
|
by combining them with those created in (1). At this stage,
|
|
we also check to make sure the variant combination is valid
|
|
in the sense that it COULD run on the platform. It may still
|
|
be undefined.
|
|
|
|
We make use of global queries to provide a thorough protection
|
|
against unwillingly scheduling tasks we very often don't want.
|
|
|
|
Note that the flags are not intersectional. This means that if you
|
|
have live_sites=True, and profile=False, you will get tasks which
|
|
have profiling available to them. However, all of those tasks must
|
|
also be live sites.
|
|
"""
|
|
expanded_categories = {}
|
|
|
|
# These global queries get applied to all of the categories. They make it
|
|
# simpler to prevent, for example, chrome tests running in the
|
|
# "Pageload desktop" category
|
|
global_queries = []
|
|
|
|
# Rather than dealing with these flags below, simply add the
|
|
# variants related to them here
|
|
if live_sites:
|
|
requested_variants.append("live-sites")
|
|
else:
|
|
global_queries.append(PerfParser.variants["live-sites"]["negation"])
|
|
if profile:
|
|
requested_variants.append("profiling")
|
|
else:
|
|
global_queries.append(PerfParser.variants["profiling"]["negation"])
|
|
|
|
if not chrome:
|
|
global_queries.append("!chrom")
|
|
|
|
# Start by expanding the variants the variants to include combinatorial
|
|
# options, searching for these tasks is "best-effort" and we can't
|
|
# guarantee all of them will have tasks selected as some may not be
|
|
# defined in the Taskcluster config files
|
|
expanded_variants = [
|
|
variant_combination
|
|
for set_size in range(len(PerfParser.variants.keys()) + 1)
|
|
for variant_combination in itertools.combinations(
|
|
list(PerfParser.variants.keys()), set_size
|
|
)
|
|
]
|
|
|
|
# Expand the test categories to show combined platforms and apps. By default,
|
|
# we'll show all desktop platforms and no variants.
|
|
for category, category_info in PerfParser.categories.items():
|
|
|
|
# Setup the platforms
|
|
for platform, platform_info in PerfParser.platforms.items():
|
|
if len(requested_platforms) > 0 and platform not in requested_platforms:
|
|
# Skip the platform because it wasn't requested
|
|
continue
|
|
|
|
platform_type = platform_info["platform"]
|
|
if not android and platform_type == "android":
|
|
# Skip android if it wasn't requested
|
|
continue
|
|
|
|
# The queries field will hold all the queries needed to run
|
|
# (in any order). Combinations of queries are used to make the
|
|
# selected tests increasingly more specific.
|
|
new_category = category + " %s" % platform
|
|
cur_cat = {
|
|
"queries": [category_info["query"]]
|
|
+ [platform_info["query"]]
|
|
+ global_queries,
|
|
"tasks": category_info["tasks"],
|
|
"platform": platform_type,
|
|
}
|
|
# If we didn't request apps, add the global category
|
|
if len(requested_apps) == 0:
|
|
expanded_categories[new_category] = cur_cat
|
|
|
|
for app, app_info in PerfParser.apps.items():
|
|
if len(requested_apps) > 0 and app not in requested_apps:
|
|
# Skip the app because it wasn't requested
|
|
continue
|
|
|
|
if app.lower() in ("chrome", "chromium", "chrome-m") and not chrome:
|
|
# Skip chrome tests if not requested
|
|
continue
|
|
if platform_type not in app_info["platforms"]:
|
|
# Ensure this app can run on this platform
|
|
continue
|
|
|
|
new_app_category = new_category + " %s" % app
|
|
expanded_categories[new_app_category] = {
|
|
"queries": cur_cat["queries"] + [app_info["query"]],
|
|
"tasks": category_info["tasks"],
|
|
"platform": platform_type,
|
|
}
|
|
|
|
# Finally, handle expanding the variants. This needs to be done
|
|
# outside the upper for-loop because variants can apply to all
|
|
# of the expanded categories that get produced there.
|
|
if len(requested_variants) > 0:
|
|
new_categories = {}
|
|
|
|
for expanded_category, info in expanded_categories.items():
|
|
for variant_combination in expanded_variants:
|
|
if not variant_combination:
|
|
continue
|
|
# Check if the combination contains the requested variant
|
|
if not any(
|
|
variant in variant_combination for variant in requested_variants
|
|
):
|
|
continue
|
|
|
|
# Ensure that this variant combination can run on this platform
|
|
runnable = True
|
|
for variant in variant_combination:
|
|
if (
|
|
info["platform"]
|
|
not in PerfParser.variants[variant]["platforms"]
|
|
):
|
|
runnable = False
|
|
break
|
|
if not runnable:
|
|
continue
|
|
|
|
# Build the category name, and setup the queries/tasks
|
|
# that it would use/select
|
|
new_variant_category = expanded_category + " %s" % "+".join(
|
|
variant_combination
|
|
)
|
|
variant_queries = [
|
|
v_info["query"]
|
|
for v, v_info in PerfParser.variants.items()
|
|
if v in variant_combination
|
|
]
|
|
new_categories[new_variant_category] = {
|
|
"queries": info["queries"] + variant_queries,
|
|
"tasks": info["tasks"],
|
|
}
|
|
|
|
# Now ensure that the queries for this new category
|
|
# don't contain negations for the variant which could
|
|
# come from the global queries
|
|
new_queries = []
|
|
for query in new_categories[new_variant_category]["queries"]:
|
|
if any(
|
|
[
|
|
query == PerfParser.variants.get(variant)["negation"]
|
|
for variant in variant_combination
|
|
]
|
|
):
|
|
# This query is a negation of one of the variants,
|
|
# exclude it
|
|
continue
|
|
new_queries.append(query)
|
|
new_categories[new_variant_category]["queries"] = new_queries
|
|
|
|
expanded_categories.update(new_categories)
|
|
|
|
return expanded_categories
|
|
|
|
def perf_push_to_try(
|
|
selected_tasks, selected_categories, queries, try_config, dry_run
|
|
):
|
|
"""Perf-specific push to try method.
|
|
|
|
This makes use of logic from the CompareParser to do something
|
|
very similar except with log redirection. We get the comparison
|
|
revisions, then use the repository object to update between revisions
|
|
and the LogProcessor for parsing out the revisions that are used
|
|
to build the Perfherder links.
|
|
"""
|
|
vcs = get_repository_object(build.topsrcdir)
|
|
compare_commit, current_revision_ref = PerfParser.get_revisions_to_run(
|
|
vcs, None
|
|
)
|
|
|
|
# Build commit message
|
|
msg = "Perf selections={} (queries={})".format(
|
|
",".join(selected_categories),
|
|
"&".join([q for q in queries if q is not None and len(q) > 0]),
|
|
)
|
|
|
|
updated = False
|
|
new_revision_treeherder = ""
|
|
base_revision_treeherder = ""
|
|
try:
|
|
# redirect_stdout allows us to feed each line into
|
|
# a processor that we can use to catch the revision
|
|
# while providing real-time output
|
|
log_processor = LogProcessor()
|
|
with redirect_stdout(log_processor):
|
|
push_to_try(
|
|
"perf",
|
|
"{msg}".format(msg=msg),
|
|
# XXX Figure out if changing `fuzzy` to `perf` will break something
|
|
try_task_config=generate_try_task_config(
|
|
"fuzzy", selected_tasks, try_config
|
|
),
|
|
stage_changes=False,
|
|
dry_run=dry_run,
|
|
closed_tree=False,
|
|
allow_log_capture=True,
|
|
)
|
|
|
|
new_revision_treeherder = log_processor.revision
|
|
|
|
if not dry_run:
|
|
vcs.update(compare_commit)
|
|
updated = True
|
|
|
|
with redirect_stdout(log_processor):
|
|
# XXX Figure out if we can use the `again` selector in some way
|
|
# Right now we would need to modify it to be able to do this.
|
|
# XXX Fix up the again selector for the perf selector (if it makes sense to)
|
|
push_to_try(
|
|
"perf-again",
|
|
"{msg}".format(msg=msg),
|
|
try_task_config=generate_try_task_config(
|
|
"fuzzy", selected_tasks, try_config
|
|
),
|
|
stage_changes=False,
|
|
dry_run=dry_run,
|
|
closed_tree=False,
|
|
allow_log_capture=True,
|
|
)
|
|
|
|
base_revision_treeherder = log_processor.revision
|
|
finally:
|
|
if updated:
|
|
vcs.update(current_revision_ref)
|
|
|
|
return base_revision_treeherder, new_revision_treeherder
|
|
|
|
def run(
|
|
update=False,
|
|
show_all=False,
|
|
android=False,
|
|
chrome=False,
|
|
live_sites=False,
|
|
parameters=None,
|
|
profile=False,
|
|
requested_variants=[],
|
|
requested_platforms=[],
|
|
requested_apps=[],
|
|
try_config=None,
|
|
dry_run=False,
|
|
**kwargs
|
|
):
|
|
# Setup fzf
|
|
fzf = fzf_bootstrap(update)
|
|
|
|
if not fzf:
|
|
print(FZF_NOT_FOUND)
|
|
return 1
|
|
|
|
all_tasks, dep_cache, cache_dir = setup_tasks_for_fzf(
|
|
not dry_run,
|
|
parameters,
|
|
full=True,
|
|
disable_target_task_filter=False,
|
|
)
|
|
base_cmd = build_base_cmd(fzf, dep_cache, cache_dir, show_estimates=False)
|
|
|
|
# Perform the selection, then push to try and return the revisions
|
|
queries = []
|
|
selected_categories = []
|
|
if not show_all:
|
|
# Expand the categories first
|
|
expanded_categories = PerfParser.expand_categories(
|
|
android=android,
|
|
chrome=chrome,
|
|
live_sites=live_sites,
|
|
profile=profile,
|
|
requested_variants=requested_variants,
|
|
requested_platforms=requested_platforms,
|
|
requested_apps=requested_apps,
|
|
)
|
|
selected_tasks, selected_categories, queries = PerfParser.get_perf_tasks(
|
|
base_cmd, all_tasks, expanded_categories
|
|
)
|
|
else:
|
|
selected_tasks = PerfParser.get_tasks(base_cmd, queries, None, all_tasks)
|
|
|
|
return PerfParser.perf_push_to_try(
|
|
selected_tasks, selected_categories, queries, try_config, dry_run
|
|
)
|
|
|
|
|
|
def run(
|
|
dry_run=False,
|
|
show_all=False,
|
|
android=False,
|
|
chrome=False,
|
|
live_sites=False,
|
|
parameters=None,
|
|
profile=False,
|
|
requested_variants=[],
|
|
requested_platforms=[],
|
|
requested_apps=[],
|
|
**kwargs
|
|
):
|
|
revisions = PerfParser.run(
|
|
dry_run=dry_run,
|
|
show_all=show_all,
|
|
android=android,
|
|
chrome=chrome,
|
|
live_sites=live_sites,
|
|
parameters=parameters,
|
|
profile=profile,
|
|
requested_variants=requested_variants,
|
|
requested_platforms=requested_platforms,
|
|
requested_apps=requested_apps,
|
|
**kwargs
|
|
)
|
|
|
|
# Provide link to perfherder for comparisons now
|
|
perfcompare_url = PERFHERDER_BASE_URL % revisions
|
|
print(
|
|
"\n!!!NOTE!!!\n You'll be able to find a performance comparison here "
|
|
"once the tests are complete (ensure you select the right "
|
|
"framework): %s\n" % perfcompare_url
|
|
)
|
|
print(
|
|
"If you need any help, you can find us in the #perf-help Element channel:\n"
|
|
"https://matrix.to/#/#perf-help:mozilla.org"
|
|
)
|
|
print(
|
|
"For more information on the performance tests, see our PerfDocs here:\n"
|
|
"https://firefox-source-docs.mozilla.org/testing/perfdocs/"
|
|
)
|