c6914e0c0a
Now that mozharness is in tree, we don't need references to mozharness repositories and revisions. We don't cleanup all mozharness stuff because android and desktop builds still references external mozharness.
501 lines
19 KiB
Python
501 lines
19 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import
|
|
|
|
import os
|
|
import json
|
|
import copy
|
|
import sys
|
|
|
|
from mach.decorators import (
|
|
CommandArgument,
|
|
CommandProvider,
|
|
Command,
|
|
)
|
|
|
|
|
|
ROOT = os.path.dirname(os.path.realpath(__file__))
|
|
GECKO = os.path.realpath(os.path.join(ROOT, '..', '..'))
|
|
DOCKER_ROOT = os.path.join(ROOT, '..', 'docker')
|
|
MOZHARNESS_CONFIG = os.path.join(GECKO, 'testing', 'mozharness', 'mozharness.json')
|
|
|
|
# XXX: If/when we have the taskcluster queue use construct url instead
|
|
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
|
|
REGISTRY = open(os.path.join(DOCKER_ROOT, 'REGISTRY')).read().strip()
|
|
|
|
DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
|
|
|
|
TREEHERDER_ROUTE_PREFIX = 'tc-treeherder-stage'
|
|
TREEHERDER_ROUTES = {
|
|
'staging': 'tc-treeherder-stage',
|
|
'production': 'tc-treeherder'
|
|
}
|
|
|
|
DEFAULT_TRY = 'try: -b do -p all -u all'
|
|
DEFAULT_JOB_PATH = os.path.join(
|
|
ROOT, 'tasks', 'branches', 'base_jobs.yml'
|
|
)
|
|
|
|
def load_mozharness_info():
|
|
with open(MOZHARNESS_CONFIG) as content:
|
|
return json.load(content)
|
|
|
|
def docker_image(name):
|
|
''' Determine the docker tag/revision from an in tree docker file '''
|
|
repository_path = os.path.join(DOCKER_ROOT, name, 'REGISTRY')
|
|
repository = REGISTRY
|
|
|
|
version = open(os.path.join(DOCKER_ROOT, name, 'VERSION')).read().strip()
|
|
|
|
if os.path.isfile(repository_path):
|
|
repository = open(repository_path).read().strip()
|
|
|
|
return '{}/{}:{}'.format(repository, name, version)
|
|
|
|
def get_task(task_id):
|
|
import urllib2
|
|
return json.load(urllib2.urlopen("https://queue.taskcluster.net/v1/task/" + task_id))
|
|
|
|
|
|
def gaia_info():
|
|
'''
|
|
Fetch details from in tree gaia.json (which links this version of
|
|
gecko->gaia) and construct the usual base/head/ref/rev pairing...
|
|
'''
|
|
gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json')))
|
|
|
|
if gaia['git'] is None or \
|
|
gaia['git']['remote'] == '' or \
|
|
gaia['git']['git_revision'] == '' or \
|
|
gaia['git']['branch'] == '':
|
|
|
|
# Just use the hg params...
|
|
return {
|
|
'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
|
|
'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
|
|
'gaia_ref': gaia['revision'],
|
|
'gaia_rev': gaia['revision']
|
|
}
|
|
|
|
else:
|
|
# Use git
|
|
return {
|
|
'gaia_base_repository': gaia['git']['remote'],
|
|
'gaia_head_repository': gaia['git']['remote'],
|
|
'gaia_rev': gaia['git']['git_revision'],
|
|
'gaia_ref': gaia['git']['branch'],
|
|
}
|
|
|
|
def decorate_task_treeherder_routes(task, suffix):
|
|
"""
|
|
Decorate the given task with treeherder routes.
|
|
|
|
Uses task.extra.treeherderEnv if available otherwise defaults to only
|
|
staging.
|
|
|
|
:param dict task: task definition.
|
|
:param str suffix: The project/revision_hash portion of the route.
|
|
"""
|
|
|
|
if 'extra' not in task:
|
|
return
|
|
|
|
if 'routes' not in task:
|
|
task['routes'] = []
|
|
|
|
treeheder_env = task['extra'].get('treeherderEnv', ['staging'])
|
|
|
|
for env in treeheder_env:
|
|
task['routes'].append('{}.{}'.format(TREEHERDER_ROUTES[env], suffix))
|
|
|
|
def configure_dependent_task(task_path, parameters, taskid, templates, build_treeherder_config):
|
|
"""
|
|
Configure a build dependent task. This is shared between post-build and test tasks.
|
|
|
|
:param task_path: location to the task yaml
|
|
:param parameters: parameters to load the template
|
|
:param taskid: taskid of the dependent task
|
|
:param templates: reference to the template builder
|
|
:param build_treeherder_config: parent treeherder config
|
|
:return: the configured task
|
|
"""
|
|
task = templates.load(task_path, parameters)
|
|
task['taskId'] = taskid
|
|
|
|
if 'requires' not in task:
|
|
task['requires'] = []
|
|
|
|
task['requires'].append(parameters['build_slugid'])
|
|
|
|
if 'treeherder' not in task['task']['extra']:
|
|
task['task']['extra']['treeherder'] = {}
|
|
|
|
# Copy over any treeherder configuration from the build so
|
|
# tests show up under the same platform...
|
|
treeherder_config = task['task']['extra']['treeherder']
|
|
|
|
treeherder_config['collection'] = \
|
|
build_treeherder_config.get('collection', {})
|
|
|
|
treeherder_config['build'] = \
|
|
build_treeherder_config.get('build', {})
|
|
|
|
treeherder_config['machine'] = \
|
|
build_treeherder_config.get('machine', {})
|
|
|
|
if 'routes' not in task['task']:
|
|
task['task']['routes'] = []
|
|
|
|
if 'scopes' not in task['task']:
|
|
task['task']['scopes'] = []
|
|
|
|
return task
|
|
|
|
@CommandProvider
|
|
class DecisionTask(object):
|
|
@Command('taskcluster-decision', category="ci",
|
|
description="Build a decision task")
|
|
@CommandArgument('--project',
|
|
required=True,
|
|
help='Treeherder project name')
|
|
@CommandArgument('--url',
|
|
required=True,
|
|
help='Gecko repository to use as head repository.')
|
|
@CommandArgument('--revision',
|
|
required=True,
|
|
help='Revision for this project')
|
|
@CommandArgument('--revision-hash',
|
|
help='Treeherder revision hash')
|
|
@CommandArgument('--comment',
|
|
required=True,
|
|
help='Commit message for this revision')
|
|
@CommandArgument('--owner',
|
|
required=True,
|
|
help='email address of who owns this graph')
|
|
@CommandArgument('task', help="Path to decision task to run.")
|
|
def run_task(self, **params):
|
|
from taskcluster_graph.slugidjar import SlugidJar
|
|
from taskcluster_graph.from_now import (
|
|
json_time_from_now,
|
|
current_json_time,
|
|
)
|
|
from taskcluster_graph.templates import Templates
|
|
|
|
templates = Templates(ROOT)
|
|
# Template parameters used when expanding the graph
|
|
parameters = dict(gaia_info().items() + {
|
|
'source': 'http://todo.com/soon',
|
|
'project': params['project'],
|
|
'comment': params['comment'],
|
|
'url': params['url'],
|
|
'revision': params['revision'],
|
|
'revision_hash': params.get('revision_hash', ''),
|
|
'owner': params['owner'],
|
|
'as_slugid': SlugidJar(),
|
|
'from_now': json_time_from_now,
|
|
'now': current_json_time()
|
|
}.items())
|
|
task = templates.load(params['task'], parameters)
|
|
print(json.dumps(task, indent=4))
|
|
|
|
@CommandProvider
|
|
class Graph(object):
|
|
@Command('taskcluster-graph', category="ci",
|
|
description="Create taskcluster task graph")
|
|
@CommandArgument('--base-repository',
|
|
default=os.environ.get('GECKO_BASE_REPOSITORY'),
|
|
help='URL for "base" repository to clone')
|
|
@CommandArgument('--head-repository',
|
|
default=os.environ.get('GECKO_HEAD_REPOSITORY'),
|
|
help='URL for "head" repository to fetch revision from')
|
|
@CommandArgument('--head-ref',
|
|
default=os.environ.get('GECKO_HEAD_REF'),
|
|
help='Reference (this is same as rev usually for hg)')
|
|
@CommandArgument('--head-rev',
|
|
default=os.environ.get('GECKO_HEAD_REV'),
|
|
help='Commit revision to use from head repository')
|
|
@CommandArgument('--message',
|
|
help='Commit message to be parsed. Example: "try: -b do -p all -u all"')
|
|
@CommandArgument('--revision-hash',
|
|
required=False,
|
|
help='Treeherder revision hash to attach results to')
|
|
@CommandArgument('--project',
|
|
required=True,
|
|
help='Project to use for creating task graph. Example: --project=try')
|
|
@CommandArgument('--pushlog-id',
|
|
dest='pushlog_id',
|
|
required=False,
|
|
default=0)
|
|
@CommandArgument('--owner',
|
|
required=True,
|
|
help='email address of who owns this graph')
|
|
@CommandArgument('--extend-graph',
|
|
action="store_true", dest="ci", help='Omit create graph arguments')
|
|
def create_graph(self, **params):
|
|
from taskcluster_graph.commit_parser import parse_commit
|
|
from taskcluster_graph.slugid import slugid
|
|
from taskcluster_graph.from_now import (
|
|
json_time_from_now,
|
|
current_json_time,
|
|
)
|
|
from taskcluster_graph.templates import Templates
|
|
import taskcluster_graph.build_task
|
|
|
|
project = params['project']
|
|
message = params.get('message', '') if project == 'try' else DEFAULT_TRY
|
|
|
|
# Message would only be blank when not created from decision task
|
|
if project == 'try' and not message:
|
|
sys.stderr.write(
|
|
"Must supply commit message when creating try graph. " \
|
|
"Example: --message='try: -b do -p all -u all'"
|
|
)
|
|
sys.exit(1)
|
|
|
|
templates = Templates(ROOT)
|
|
job_path = os.path.join(ROOT, 'tasks', 'branches', project, 'job_flags.yml')
|
|
job_path = job_path if os.path.exists(job_path) else DEFAULT_JOB_PATH
|
|
|
|
jobs = templates.load(job_path, {})
|
|
|
|
job_graph = parse_commit(message, jobs)
|
|
mozharness = load_mozharness_info()
|
|
|
|
# Template parameters used when expanding the graph
|
|
parameters = dict(gaia_info().items() + {
|
|
'project': project,
|
|
'pushlog_id': params.get('pushlog_id', 0),
|
|
'docker_image': docker_image,
|
|
'base_repository': params['base_repository'] or \
|
|
params['head_repository'],
|
|
'head_repository': params['head_repository'],
|
|
'head_ref': params['head_ref'] or params['head_rev'],
|
|
'head_rev': params['head_rev'],
|
|
'owner': params['owner'],
|
|
'from_now': json_time_from_now,
|
|
'now': current_json_time(),
|
|
'mozharness_repository': mozharness['repo'],
|
|
'mozharness_rev': mozharness['revision'],
|
|
'mozharness_ref':mozharness.get('reference', mozharness['revision']),
|
|
'revision_hash': params['revision_hash']
|
|
}.items())
|
|
|
|
treeherder_route = '{}.{}'.format(
|
|
params['project'],
|
|
params.get('revision_hash', '')
|
|
)
|
|
|
|
# Task graph we are generating for taskcluster...
|
|
graph = {
|
|
'tasks': [],
|
|
'scopes': []
|
|
}
|
|
|
|
if params['revision_hash']:
|
|
for env in TREEHERDER_ROUTES:
|
|
graph['scopes'].append('queue:route:{}.{}'.format(TREEHERDER_ROUTES[env], treeherder_route))
|
|
|
|
graph['metadata'] = {
|
|
'source': 'http://todo.com/what/goes/here',
|
|
'owner': params['owner'],
|
|
# TODO: Add full mach commands to this example?
|
|
'description': 'Task graph generated via ./mach taskcluster-graph',
|
|
'name': 'task graph local'
|
|
}
|
|
|
|
for build in job_graph:
|
|
build_parameters = dict(parameters)
|
|
build_parameters['build_slugid'] = slugid()
|
|
build_task = templates.load(build['task'], build_parameters)
|
|
|
|
if 'routes' not in build_task['task']:
|
|
build_task['task']['routes'] = []
|
|
|
|
if params['revision_hash']:
|
|
decorate_task_treeherder_routes(build_task['task'],
|
|
treeherder_route)
|
|
|
|
# Ensure each build graph is valid after construction.
|
|
taskcluster_graph.build_task.validate(build_task)
|
|
graph['tasks'].append(build_task)
|
|
|
|
test_packages_url, tests_url = None, None
|
|
|
|
if 'test_packages' in build_task['task']['extra']['locations']:
|
|
test_packages_url = ARTIFACT_URL.format(
|
|
build_parameters['build_slugid'],
|
|
build_task['task']['extra']['locations']['test_packages']
|
|
)
|
|
|
|
if 'tests' in build_task['task']['extra']['locations']:
|
|
tests_url = ARTIFACT_URL.format(
|
|
build_parameters['build_slugid'],
|
|
build_task['task']['extra']['locations']['tests']
|
|
)
|
|
|
|
build_url = ARTIFACT_URL.format(
|
|
build_parameters['build_slugid'],
|
|
build_task['task']['extra']['locations']['build']
|
|
)
|
|
|
|
# img_url is only necessary for device builds
|
|
img_url = ARTIFACT_URL.format(
|
|
build_parameters['build_slugid'],
|
|
build_task['task']['extra']['locations'].get('img', '')
|
|
)
|
|
|
|
define_task = DEFINE_TASK.format(build_task['task']['workerType'])
|
|
|
|
graph['scopes'].append(define_task)
|
|
graph['scopes'].extend(build_task['task'].get('scopes', []))
|
|
route_scopes = map(lambda route: 'queue:route:' + route, build_task['task'].get('routes', []))
|
|
graph['scopes'].extend(route_scopes)
|
|
|
|
# Treeherder symbol configuration for the graph required for each
|
|
# build so tests know which platform they belong to.
|
|
build_treeherder_config = build_task['task']['extra']['treeherder']
|
|
|
|
if 'machine' not in build_treeherder_config:
|
|
message = '({}), extra.treeherder.machine required for all builds'
|
|
raise ValueError(message.format(build['task']))
|
|
|
|
if 'build' not in build_treeherder_config:
|
|
build_treeherder_config['build'] = \
|
|
build_treeherder_config['machine']
|
|
|
|
if 'collection' not in build_treeherder_config:
|
|
build_treeherder_config['collection'] = { 'opt': True }
|
|
|
|
if len(build_treeherder_config['collection'].keys()) != 1:
|
|
message = '({}), extra.treeherder.collection must contain one type'
|
|
raise ValueError(message.fomrat(build['task']))
|
|
|
|
for post_build in build['post-build']:
|
|
# copy over the old parameters to update the template
|
|
post_parameters = copy.copy(build_parameters)
|
|
post_task = configure_dependent_task(post_build['task'],
|
|
post_parameters,
|
|
slugid(),
|
|
templates,
|
|
build_treeherder_config)
|
|
graph['tasks'].append(post_task)
|
|
|
|
for test in build['dependents']:
|
|
test = test['allowed_build_tasks'][build['task']]
|
|
test_parameters = copy.copy(build_parameters)
|
|
test_parameters['build_url'] = build_url
|
|
test_parameters['img_url'] = img_url
|
|
if tests_url:
|
|
test_parameters['tests_url'] = tests_url
|
|
if test_packages_url:
|
|
test_parameters['test_packages_url'] = test_packages_url
|
|
test_definition = templates.load(test['task'], {})['task']
|
|
chunk_config = test_definition['extra']['chunks']
|
|
|
|
# Allow branch configs to override task level chunking...
|
|
if 'chunks' in test:
|
|
chunk_config['total'] = test['chunks']
|
|
|
|
test_parameters['total_chunks'] = chunk_config['total']
|
|
|
|
for chunk in range(1, chunk_config['total'] + 1):
|
|
if 'only_chunks' in test and \
|
|
chunk not in test['only_chunks']:
|
|
continue
|
|
|
|
test_parameters['chunk'] = chunk
|
|
test_task = configure_dependent_task(test['task'],
|
|
test_parameters,
|
|
slugid(),
|
|
templates,
|
|
build_treeherder_config)
|
|
|
|
if params['revision_hash']:
|
|
decorate_task_treeherder_routes(
|
|
test_task['task'], treeherder_route)
|
|
|
|
graph['tasks'].append(test_task)
|
|
|
|
define_task = DEFINE_TASK.format(
|
|
test_task['task']['workerType']
|
|
)
|
|
|
|
graph['scopes'].append(define_task)
|
|
graph['scopes'].extend(test_task['task'].get('scopes', []))
|
|
|
|
graph['scopes'] = list(set(graph['scopes']))
|
|
|
|
# When we are extending the graph remove extra fields...
|
|
if params['ci'] is True:
|
|
graph.pop('scopes', None)
|
|
graph.pop('metadata', None)
|
|
|
|
print(json.dumps(graph, indent=4))
|
|
|
|
@CommandProvider
|
|
class CIBuild(object):
|
|
@Command('taskcluster-build', category='ci',
|
|
description="Create taskcluster try server build task")
|
|
@CommandArgument('--base-repository',
|
|
help='URL for "base" repository to clone')
|
|
@CommandArgument('--head-repository',
|
|
required=True,
|
|
help='URL for "head" repository to fetch revision from')
|
|
@CommandArgument('--head-ref',
|
|
help='Reference (this is same as rev usually for hg)')
|
|
@CommandArgument('--head-rev',
|
|
required=True,
|
|
help='Commit revision to use')
|
|
@CommandArgument('--owner',
|
|
default='foobar@mozilla.com',
|
|
help='email address of who owns this graph')
|
|
@CommandArgument('build_task',
|
|
help='path to build task definition')
|
|
def create_ci_build(self, **params):
|
|
from taskcluster_graph.templates import Templates
|
|
import taskcluster_graph.build_task
|
|
|
|
templates = Templates(ROOT)
|
|
# TODO handle git repos
|
|
head_repository = params['head_repository']
|
|
if not head_repository:
|
|
head_repository = get_hg_url()
|
|
|
|
head_rev = params['head_rev']
|
|
if not head_rev:
|
|
head_rev = get_latest_hg_revision(head_repository)
|
|
|
|
head_ref = params['head_ref'] or head_rev
|
|
|
|
from taskcluster_graph.from_now import (
|
|
json_time_from_now,
|
|
current_json_time,
|
|
)
|
|
build_parameters = dict(gaia_info().items() + {
|
|
'docker_image': docker_image,
|
|
'owner': params['owner'],
|
|
'from_now': json_time_from_now,
|
|
'now': current_json_time(),
|
|
'base_repository': params['base_repository'] or head_repository,
|
|
'head_repository': head_repository,
|
|
'head_rev': head_rev,
|
|
'head_ref': head_ref,
|
|
}.items())
|
|
|
|
try:
|
|
build_task = templates.load(params['build_task'], build_parameters)
|
|
except IOError:
|
|
sys.stderr.write(
|
|
"Could not load build task file. Ensure path is a relative " \
|
|
"path from testing/taskcluster"
|
|
)
|
|
sys.exit(1)
|
|
|
|
taskcluster_graph.build_task.validate(build_task)
|
|
|
|
print(json.dumps(build_task['task'], indent=4))
|