Bug 1258497: Factor out some common code; r=gps
This code will also be used by taskgraph's legacy kind. MozReview-Commit-ID: Gd4zZYEee2W
This commit is contained in:
Dustin J. Mitchell
@@ -38,164 +38,6 @@ DEFAULT_JOB_PATH = os.path.join(
|
||||
# time after which a try build's results will expire
|
||||
TRY_EXPIRATION = "14 days"
|
||||
|
||||
def merge_dicts(*dicts):
|
||||
merged_dict = {}
|
||||
for dictionary in dicts:
|
||||
merged_dict.update(dictionary)
|
||||
return merged_dict
|
||||
|
||||
def gaia_info():
|
||||
'''
|
||||
Fetch details from in tree gaia.json (which links this version of
|
||||
gecko->gaia) and construct the usual base/head/ref/rev pairing...
|
||||
'''
|
||||
gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json')))
|
||||
|
||||
if gaia['git'] is None or \
|
||||
gaia['git']['remote'] == '' or \
|
||||
gaia['git']['git_revision'] == '' or \
|
||||
gaia['git']['branch'] == '':
|
||||
|
||||
# Just use the hg params...
|
||||
return {
|
||||
'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
|
||||
'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
|
||||
'gaia_ref': gaia['revision'],
|
||||
'gaia_rev': gaia['revision']
|
||||
}
|
||||
|
||||
else:
|
||||
# Use git
|
||||
return {
|
||||
'gaia_base_repository': gaia['git']['remote'],
|
||||
'gaia_head_repository': gaia['git']['remote'],
|
||||
'gaia_rev': gaia['git']['git_revision'],
|
||||
'gaia_ref': gaia['git']['branch'],
|
||||
}
|
||||
|
||||
def configure_dependent_task(task_path, parameters, taskid, templates, build_treeherder_config):
|
||||
"""
|
||||
Configure a build dependent task. This is shared between post-build and test tasks.
|
||||
|
||||
:param task_path: location to the task yaml
|
||||
:param parameters: parameters to load the template
|
||||
:param taskid: taskid of the dependent task
|
||||
:param templates: reference to the template builder
|
||||
:param build_treeherder_config: parent treeherder config
|
||||
:return: the configured task
|
||||
"""
|
||||
task = templates.load(task_path, parameters)
|
||||
task['taskId'] = taskid
|
||||
|
||||
if 'requires' not in task:
|
||||
task['requires'] = []
|
||||
|
||||
task['requires'].append(parameters['build_slugid'])
|
||||
|
||||
if 'treeherder' not in task['task']['extra']:
|
||||
task['task']['extra']['treeherder'] = {}
|
||||
|
||||
# Copy over any treeherder configuration from the build so
|
||||
# tests show up under the same platform...
|
||||
treeherder_config = task['task']['extra']['treeherder']
|
||||
|
||||
treeherder_config['collection'] = \
|
||||
build_treeherder_config.get('collection', {})
|
||||
|
||||
treeherder_config['build'] = \
|
||||
build_treeherder_config.get('build', {})
|
||||
|
||||
treeherder_config['machine'] = \
|
||||
build_treeherder_config.get('machine', {})
|
||||
|
||||
if 'routes' not in task['task']:
|
||||
task['task']['routes'] = []
|
||||
|
||||
if 'scopes' not in task['task']:
|
||||
task['task']['scopes'] = []
|
||||
|
||||
return task
|
||||
|
||||
def set_interactive_task(task, interactive):
|
||||
r"""Make the task interactive.
|
||||
|
||||
:param task: task definition.
|
||||
:param interactive: True if the task should be interactive.
|
||||
"""
|
||||
if not interactive:
|
||||
return
|
||||
|
||||
payload = task["task"]["payload"]
|
||||
if "features" not in payload:
|
||||
payload["features"] = {}
|
||||
payload["features"]["interactive"] = True
|
||||
|
||||
def remove_caches_from_task(task):
|
||||
r"""Remove all caches but tc-vcs from the task.
|
||||
|
||||
:param task: task definition.
|
||||
"""
|
||||
whitelist = [
|
||||
re.compile("^level-[123]-.*-tc-vcs(-public-sources)?$"),
|
||||
re.compile("^tooltool-cache$"),
|
||||
]
|
||||
try:
|
||||
caches = task["task"]["payload"]["cache"]
|
||||
for cache in caches.keys():
|
||||
if not any(pat.match(cache) for pat in whitelist):
|
||||
caches.pop(cache)
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def set_expiration(task, timestamp):
|
||||
task_def = task['task']
|
||||
task_def['expires'] = timestamp
|
||||
if task_def.get('deadline', timestamp) > timestamp:
|
||||
task_def['deadline'] = timestamp
|
||||
|
||||
try:
|
||||
artifacts = task_def['payload']['artifacts']
|
||||
except KeyError:
|
||||
return
|
||||
|
||||
for artifact in artifacts.values():
|
||||
artifact['expires'] = timestamp
|
||||
|
||||
def query_vcs_info(repository, revision):
|
||||
"""Query the pushdate and pushid of a repository/revision.
|
||||
This is intended to be used on hg.mozilla.org/mozilla-central and
|
||||
similar. It may or may not work for other hg repositories.
|
||||
"""
|
||||
if not repository or not revision:
|
||||
sys.stderr.write('cannot query vcs info because vcs info not provided\n')
|
||||
return None
|
||||
|
||||
VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets'])
|
||||
|
||||
try:
|
||||
import requests
|
||||
url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'),
|
||||
revision)
|
||||
sys.stderr.write("Querying version control for metadata: %s\n" % url)
|
||||
contents = requests.get(url).json()
|
||||
|
||||
changesets = []
|
||||
for c in contents['changesets']:
|
||||
changesets.append({k: c[k] for k in ('desc', 'files', 'node')})
|
||||
|
||||
pushid = contents['changesets'][-1]['pushid']
|
||||
pushdate = contents['changesets'][-1]['pushdate'][0]
|
||||
|
||||
return VCSInfo(pushid, pushdate, changesets)
|
||||
|
||||
except Exception:
|
||||
sys.stderr.write(
|
||||
"Error querying VCS info for '%s' revision '%s'\n" % (
|
||||
repository, revision,
|
||||
)
|
||||
)
|
||||
return None
|
||||
|
||||
@CommandProvider
|
||||
class DecisionTask(object):
|
||||
@Command('taskcluster-decision', category="ci",
|
||||
@@ -219,6 +61,7 @@ class DecisionTask(object):
|
||||
help='email address of who owns this graph')
|
||||
@CommandArgument('task', help="Path to decision task to run.")
|
||||
def run_task(self, **params):
|
||||
from taskcluster_graph.mach_util import gaia_info
|
||||
from taskcluster_graph.slugidjar import SlugidJar
|
||||
from taskcluster_graph.from_now import (
|
||||
json_time_from_now,
|
||||
@@ -335,6 +178,14 @@ class Graph(object):
|
||||
|
||||
from slugid import nice as slugid
|
||||
|
||||
from taskcluster_graph.mach_util import (
|
||||
merge_dicts,
|
||||
gaia_info,
|
||||
configure_dependent_task,
|
||||
set_interactive_task,
|
||||
remove_caches_from_task,
|
||||
query_vcs_info
|
||||
)
|
||||
import taskcluster_graph.transform.routes as routes_transform
|
||||
import taskcluster_graph.transform.treeherder as treeherder_transform
|
||||
from taskcluster_graph.commit_parser import parse_commit
|
||||
@@ -717,6 +568,11 @@ class CIBuild(object):
|
||||
dest="interactive",
|
||||
help="Run the task with the interactive feature enabled")
|
||||
def create_ci_build(self, **params):
|
||||
from taskcluster_graph.mach_util import (
|
||||
gaia_info,
|
||||
set_interactive_task,
|
||||
query_vcs_info
|
||||
)
|
||||
from taskcluster_graph.templates import Templates
|
||||
from taskcluster_graph.image_builder import docker_image
|
||||
import taskcluster_graph.build_task
|
||||
|
||||
Reference in New Issue
Block a user