138 lines
5.1 KiB
Python
138 lines
5.1 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
import logging
|
|
import itertools
|
|
|
|
from . import base
|
|
from .. import files_changed
|
|
from ..util.python_path import find_object
|
|
from ..util.templates import merge
|
|
from ..util.yaml import load_yaml
|
|
from ..util.seta import is_low_value_task
|
|
|
|
from ..transforms.base import TransformSequence, TransformConfig
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def get_inputs(kind, path, config, params, loaded_tasks):
|
|
"""
|
|
Get the input elements that will be transformed into tasks. The
|
|
elements themselves are free-form, and become the input to the first
|
|
transform.
|
|
|
|
By default, this reads jobs from the `jobs` key, or from yaml files
|
|
named by `jobs-from`. The entities are read from mappings, and the
|
|
keys to those mappings are added in the `name` key of each entity.
|
|
|
|
If there is a `job-defaults` config, then every job is merged with it.
|
|
This provides a simple way to set default values for all jobs of a
|
|
kind. More complex defaults should be implemented with custom
|
|
transforms.
|
|
|
|
Other kind implementations can use a different get_inputs function to
|
|
produce inputs and hand them to `transform_inputs`.
|
|
"""
|
|
def jobs():
|
|
defaults = config.get('job-defaults')
|
|
jobs = config.get('jobs', {}).iteritems()
|
|
jobs_from = itertools.chain.from_iterable(
|
|
load_yaml(path, filename).iteritems()
|
|
for filename in config.get('jobs-from', {}))
|
|
for name, job in itertools.chain(jobs, jobs_from):
|
|
if defaults:
|
|
job = merge(defaults, job)
|
|
yield name, job
|
|
|
|
for name, job in jobs():
|
|
job['name'] = name
|
|
logger.debug("Generating tasks for {} {}".format(kind, name))
|
|
yield job
|
|
|
|
|
|
def transform_inputs(inputs, kind, path, config, params, loaded_tasks):
|
|
"""
|
|
Transform a sequence of inputs according to the transform configuration.
|
|
"""
|
|
transforms = TransformSequence()
|
|
for xform_path in config['transforms']:
|
|
transform = find_object(xform_path)
|
|
transforms.add(transform)
|
|
|
|
# perform the transformations
|
|
trans_config = TransformConfig(kind, path, config, params)
|
|
tasks = [TransformTask(kind, t) for t in transforms(trans_config, inputs)]
|
|
return tasks
|
|
|
|
|
|
def load_tasks(kind, path, config, params, loaded_tasks):
|
|
return transform_inputs(
|
|
get_inputs(kind, path, config, params, loaded_tasks),
|
|
kind, path, config, params, loaded_tasks)
|
|
|
|
|
|
class TransformTask(base.Task):
|
|
"""
|
|
Tasks of this class are generated by applying transformations to a sequence
|
|
of input entities. By default, it gets those inputs from YAML data in the
|
|
kind directory, but subclasses may override `get_inputs` to produce them in
|
|
some other way.
|
|
"""
|
|
|
|
def __init__(self, kind, task):
|
|
self.dependencies = task.get('dependencies', {})
|
|
self.when = task.get('when', {})
|
|
super(TransformTask, self).__init__(kind, task['label'],
|
|
task['attributes'], task['task'],
|
|
index_paths=task.get('index-paths'))
|
|
|
|
def get_dependencies(self, taskgraph):
|
|
return [(label, name) for name, label in self.dependencies.items()]
|
|
|
|
def optimize(self, params):
|
|
bbb_task = False
|
|
|
|
if self.index_paths:
|
|
optimized, taskId = super(TransformTask, self).optimize(params)
|
|
if optimized:
|
|
return optimized, taskId
|
|
|
|
elif 'files-changed' in self.when:
|
|
changed = files_changed.check(
|
|
params, self.when['files-changed'])
|
|
if not changed:
|
|
logger.debug('no files found matching a pattern in `when.files-changed` for ' +
|
|
self.label)
|
|
return True, None
|
|
|
|
# no need to call SETA for build jobs
|
|
if self.task.get('extra', {}).get('treeherder', {}).get('jobKind', '') == 'build':
|
|
return False, None
|
|
|
|
# for bbb tasks we need to send in the buildbot buildername
|
|
if self.task.get('provisionerId', '') == 'buildbot-bridge':
|
|
self.label = self.task.get('payload').get('buildername')
|
|
bbb_task = True
|
|
|
|
# we would like to return 'False, None' while it's high_value_task
|
|
# and we wouldn't optimize it. Otherwise, it will return 'True, None'
|
|
if is_low_value_task(self.label,
|
|
params.get('project'),
|
|
params.get('pushlog_id'),
|
|
params.get('pushdate'),
|
|
bbb_task):
|
|
# Always optimize away low-value tasks
|
|
return True, None
|
|
else:
|
|
return False, None
|
|
|
|
@classmethod
|
|
def from_json(cls, task_dict):
|
|
# when reading back from JSON, we lose the "when" information
|
|
task_dict['when'] = {}
|
|
return cls(task_dict['attributes']['kind'], task_dict)
|