# -*- coding: utf-8 -*- # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os import os.path import json import copy import datetime import subprocess import sys import urllib2 from mach.decorators import ( CommandArgument, CommandProvider, Command, ) from taskcluster_graph.commit_parser import parse_commit from taskcluster_graph.slugid import slugid from taskcluster_graph.from_now import json_time_from_now, current_json_time from taskcluster_graph.templates import Templates import taskcluster_graph.build_task ROOT = os.path.dirname(os.path.realpath(__file__)) DOCKER_ROOT = os.path.join(ROOT, '..', 'docker') # XXX: If/when we have the taskcluster queue use construct url instead ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' REGISTRY = open(os.path.join(DOCKER_ROOT, 'REGISTRY')).read().strip() DEFINE_TASK = 'queue:define-task:aws-provisioner/{}' def get_hg_url(): ''' Determine the url for the mercurial repository''' try: url = subprocess.check_output( ['hg', 'path', 'default'], stderr=subprocess.PIPE ) except subprocess.CalledProcessError: sys.stderr.write( "Error: Could not determine the current hg repository url. " \ "Ensure command is executed within a hg respository" ) sys.exit(1) return url def get_latest_hg_revision(repository): ''' Retrieves the revision number of the latest changed head''' try: revision = subprocess.check_output( ['hg', 'id', '-r', 'tip', repository, '-i'], stderr=subprocess.PIPE ).strip('\n') except subprocess.CalledProcessError: sys.stderr.write( "Error: Could not determine the latest hg revision at {} " \ "Ensure command is executed within a cloned hg respository and " \ "remote default remote repository is accessible".format(repository) ) sys.exit(1) return revision def docker_image(name): ''' Determine the docker tag/revision from an in tree docker file ''' repository_path = os.path.join(DOCKER_ROOT, name, 'REPOSITORY') repository = REGISTRY version = open(os.path.join(DOCKER_ROOT, name, 'VERSION')).read().strip() if os.path.isfile(repository_path): repository = open(repository_path).read().strip() return '{}/{}:{}'.format(repository, name, version) def get_task(task_id): return json.load(urllib2.urlopen("https://queue.taskcluster.net/v1/task/" + task_id)) @CommandProvider class InheritTryme(object): @Command('taskcluster-inherit', category="ci", description="Create taskcluster try server graph") def tryme(self): yaml.add_platform print('meme') @CommandProvider class TryGraph(object): @Command('taskcluster-trygraph', category="ci", description="Create taskcluster try server graph") @CommandArgument('--base-repository', help='URL for "base" repository to clone') @CommandArgument('--head-repository', required=True, help='URL for "base" repository to clone') @CommandArgument('--head-ref', help='Reference (this is same as rev usually for hg)') @CommandArgument('--head-rev', required=True, help='Commit revision to use') @CommandArgument('--message', required=True, help='Commit message to be parsed') @CommandArgument('--owner', required=True, help='email address of who owns this graph') @CommandArgument('--extend-graph', action="store_true", dest="ci", help='Omit create graph arguments') def create_graph(self, **params): templates = Templates(ROOT) jobs = templates.load('job_flags.yml', {}) job_graph = parse_commit(params['message'], jobs) # Template parameters used when expanding the graph parameters = { 'docker_image': docker_image, 'base_repository': params['base_repository'] or \ params['head_repository'], 'head_repository': params['head_repository'], 'head_ref': params['head_ref'] or params['head_rev'], 'head_rev': params['head_rev'], 'owner': params['owner'], 'from_now': json_time_from_now, 'now': datetime.datetime.now().isoformat() } # Task graph we are generating for taskcluster... graph = { 'tasks': [], 'scopes': [] } if params['ci'] is False: graph['metadata'] = { 'source': 'http://todo.com/what/goes/here', 'owner': params['owner'], # TODO: Add full mach commands to this example? 'description': 'Try task graph generated via ./mach trygraph', 'name': 'trygraph local' } for build in job_graph: build_parameters = dict(parameters, **build['additional-parameters']) build_parameters['build_slugid'] = slugid() build_task = templates.load(build['task'], build_parameters) # Ensure each build graph is valid after construction. taskcluster_graph.build_task.validate(build_task) graph['tasks'].append(build_task) tests_url = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations']['tests'] ) build_url = ARTIFACT_URL.format( build_parameters['build_slugid'], build_task['task']['extra']['locations']['build'] ) define_task = DEFINE_TASK.format(build_task['task']['workerType']) graph['scopes'].append(define_task) graph['scopes'].extend(build_task['task'].get('scopes', [])) for test in build['dependents']: test = test['allowed_build_tasks'][build['task']] test_parameters = copy.copy(build_parameters) test_parameters['build_url'] = build_url test_parameters['tests_url'] = tests_url test_parameters['total_chunks'] = 1 if 'chunks' in test: test_parameters['total_chunks'] = test['chunks'] for chunk in range(1, test_parameters['total_chunks'] + 1): test_parameters['chunk'] = chunk test_task = templates.load(test['task'], test_parameters) test_task['taskId'] = slugid() if 'requires' not in test_task: test_task['requires'] = [] test_task['requires'].append(test_parameters['build_slugid']) graph['tasks'].append(test_task) define_task = DEFINE_TASK.format( test_task['task']['workerType'] ) graph['scopes'].append(define_task) graph['scopes'].extend(test_task['task'].get('scopes', [])) graph['scopes'] = list(set(graph['scopes'])) print(json.dumps(graph, indent=4)) @CommandProvider class CIBuild(object): @Command('taskcluster-build', category='ci', description="Create taskcluster try server build task") @CommandArgument('--b2g-config', help='(emulators/phones only) in tree build configuration directory') @CommandArgument('--debug', action='store_true', help='(emulators/phones only) build debug images') @CommandArgument('--base-repository', help='URL for "base" repository to clone') @CommandArgument('--head-repository', required=True, help='URL for "base" repository to clone') @CommandArgument('--head-ref', help='Reference (this is same as rev usually for hg)') @CommandArgument('--head-rev', required=True, help='Commit revision to use') @CommandArgument('--owner', help='email address of who owns this graph') @CommandArgument('build_task', help='path to build task definition') def create_ci_build(self, **params): templates = Templates(ROOT) # TODO handle git repos head_repository = params['head_repository'] if not head_repository: head_repository = get_hg_url() head_rev = params['head_rev'] if not head_rev: head_rev = get_latest_hg_revision(head_repository) head_ref = params['head_ref'] or head_rev debug = 1 if params.get('debug') else 0 build_parameters = { 'docker_image': docker_image, 'b2g-config': params['b2g_config'], 'debug': debug, 'build-type': 'Debug' if debug else 'Opt', 'owner': params['owner'], 'from_now': json_time_from_now, 'now': current_json_time(), 'base_repository': params['base_repository'] or head_repository, 'head_repository': head_repository, 'head_rev': head_rev, 'head_ref': head_ref } try: build_task = templates.load(params['build_task'], build_parameters) except IOError: sys.stderr.write( "Could not load build task file. Ensure path is a relative " \ "path from testing/taskcluster" ) sys.exit(1) taskcluster_graph.build_task.validate(build_task) print(json.dumps(build_task['task'], indent=4)) @CommandProvider class CITest(object): @Command('taskcluster-test', category='ci', description='Create taskcluster try server test task') @CommandArgument('--task-id', help='the task id to pick the correct build and tests') @CommandArgument('--total-chunks', type=int, help='total number of chunks') @CommandArgument('--chunk', type=int, help='current chunk') @CommandArgument('--owner', help='email address of who owns this graph') @CommandArgument('test_task', help='path to the test task definition') def create_ci_test(self, test_task, task_id='', total_chunks=1, chunk=1, owner=''): if total_chunks is None: total_chunks = 1 if chunk is None: chunk = 1 if chunk < 1 or chunk > total_chunks: raise ValueError( '"chunk" must be a value between 1 and "total_chunks (default 1)"') build_url, tests_url = self._get_build_and_tests_url(task_id) test_parameters = { 'docker_image': docker_image, 'build_url': ARTIFACT_URL.format(task_id, build_url), 'tests_url': ARTIFACT_URL.format(task_id, tests_url), 'total_chunks': total_chunks, 'chunk': chunk, 'owner': owner, 'from_now': json_time_from_now, 'now': current_json_time() } try: test_task = import_yaml(test_task, test_parameters) except IOError: sys.stderr.write( "Could not load test task file. Ensure path is a relative " \ "path from testing/taskcluster" ) sys.exit(1) print(json.dumps(test_task['task'], indent=4)) def _get_build_and_tests_url(self, task_id): task = get_task(task_id) locations = task['extra']['locations'] return locations['build'], locations['tests'] @CommandProvider class CIDockerRun(object): @Command('taskcluster-docker-run', category='ci', description='Run a docker image and optionally mount local hg repos. ' \ 'Repos will be mounted to /home/worker/x/source accordingly. ' \ 'For example, to run a centos image and mount local gecko ' \ 'and gaia repos: mach ci-docker-run --local-gecko-repo ' \ '/home/user/mozilla-central/ --local-gaia-repo /home/user/gaia/ '\ '--docker-flags="-t -i" centos:centos7 /bin/bash') @CommandArgument('--local-gecko-repo', action='store', dest='local_gecko_repo', help='local gecko hg repository for volume mount') @CommandArgument('--gecko-revision', action='store', dest='gecko_revision', help='local gecko repo revision (defaults to latest)') @CommandArgument('--local-gaia-repo', action='store', dest='local_gaia_repo', help='local gaia hg repository for volume mount') @CommandArgument('--mozconfig', help='The mozconfig file for building gecko') @CommandArgument('--docker-flags', action='store', dest='flags', help='string of run flags (i.e. --docker-flags="-i -t")') @CommandArgument('image', help='name of docker image to run') @CommandArgument('command', nargs='*', help='command to run inside the docker image') def ci_docker_run(self, local_gecko_repo='', gecko_revision='', local_gaia_repo='', mozconfig="", flags="", **kwargs): ''' Run docker image and optionally volume mount specified local repos ''' gecko_mount_point='/home/worker/mozilla-central/source/' gaia_mount_point='/home/worker/gaia/source/' cmd_out = ['docker', 'run'] if flags: cmd_out.extend(flags.split()) if local_gecko_repo: if not os.path.exists(local_gecko_repo): print("Gecko repository path doesn't exist: %s" % local_gecko_repo) sys.exit(1) if not gecko_revision: gecko_revision = get_latest_hg_revision(local_gecko_repo) cmd_out.extend(['-v', '%s:%s' % (local_gecko_repo, gecko_mount_point)]) cmd_out.extend(['-e', 'REPOSITORY=%s' % gecko_mount_point]) cmd_out.extend(['-e', 'REVISION=%s' % gecko_revision]) if local_gaia_repo: if not os.path.exists(local_gaia_repo): print("Gaia repository path doesn't exist: %s" % local_gaia_repo) sys.exit(1) cmd_out.extend(['-v', '%s:%s' % (local_gaia_repo, gaia_mount_point)]) cmd_out.extend(['-e', 'GAIA_REPOSITORY=%s' % gaia_mount_point]) if mozconfig: cmd_out.extend(['-e', 'MOZCONFIG=%s' % mozconfig]) cmd_out.append(kwargs['image']) for cmd_x in kwargs['command']: cmd_out.append(cmd_x) try: subprocess.check_call(cmd_out) except subprocess.CalledProcessError: sys.stderr.write("Docker run command returned non-zero status. Attempted:\n") cmd_line = '' for x in cmd_out: cmd_line = cmd_line + x + ' ' sys.stderr.write(cmd_line + '\n') sys.exit(1)