5d56d07832
Previously, we ran a single "target task" function to mutate the full task graph into a subset based on input parameters (try syntax, repository being built for, etc). This concept is useful. But the implementation was limiting because we could only have a single "target tasks" function. This commit introduces the concept of "filters." They conceptually do the same thing as "target tasks methods" but you can run more than 1 of them. Filters are simply functions that examine an input graph+parameters and emit nodes that should be retained. Filters, like target tasks methods, are defined via decorated functions in a module. TaskGraphGenerator has been converted to use filters. The list of defined filters can be defined in the parameters dict passed into TaskGraphGenerator. A default filter list is provided in decision.py. The intent is to eventually convert target tasks to filters. Until that happens, we always run the registered target tasks method via a filter proxy function. No new tests have been added because we don't yet have any functionality relying explicitly on filters. Tests will be added in a subsequent commit once we add a new filter. While I was here, I also snuck in some logging on the size of the graphs. MozReview-Commit-ID: ERn2hIYbMRp
74 lines
2.0 KiB
Python
74 lines
2.0 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
import json
|
|
import yaml
|
|
from mozbuild.util import ReadOnlyDict
|
|
|
|
# Please keep this list sorted and in sync with taskcluster/docs/parameters.rst
|
|
PARAMETER_NAMES = set([
|
|
'base_repository',
|
|
'build_date',
|
|
'filters',
|
|
'head_ref',
|
|
'head_repository',
|
|
'head_rev',
|
|
'level',
|
|
'message',
|
|
'moz_build_date',
|
|
'optimize_target_tasks',
|
|
'owner',
|
|
'project',
|
|
'pushdate',
|
|
'pushlog_id',
|
|
'target_tasks_method',
|
|
'triggered_by',
|
|
])
|
|
|
|
|
|
class Parameters(ReadOnlyDict):
|
|
"""An immutable dictionary with nicer KeyError messages on failure"""
|
|
def check(self):
|
|
names = set(self)
|
|
msg = []
|
|
|
|
missing = PARAMETER_NAMES - names
|
|
if missing:
|
|
msg.append("missing parameters: " + ", ".join(missing))
|
|
|
|
extra = names - PARAMETER_NAMES
|
|
if extra:
|
|
msg.append("extra parameters: " + ", ".join(extra))
|
|
|
|
if msg:
|
|
raise Exception("; ".join(msg))
|
|
|
|
def __getitem__(self, k):
|
|
if k not in PARAMETER_NAMES:
|
|
raise KeyError("no such parameter {!r}".format(k))
|
|
try:
|
|
return super(Parameters, self).__getitem__(k)
|
|
except KeyError:
|
|
raise KeyError("taskgraph parameter {!r} not found".format(k))
|
|
|
|
|
|
def load_parameters_file(options):
|
|
"""
|
|
Load parameters from the --parameters option
|
|
"""
|
|
filename = options['parameters']
|
|
if not filename:
|
|
return Parameters()
|
|
with open(filename) as f:
|
|
if filename.endswith('.yml'):
|
|
return Parameters(**yaml.safe_load(f))
|
|
elif filename.endswith('.json'):
|
|
return Parameters(**json.load(f))
|
|
else:
|
|
raise TypeError("Parameters file `{}` is not JSON or YAML".format(filename))
|