218 lines
7.5 KiB
Python
218 lines
7.5 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import, print_function, unicode_literals
|
|
|
|
import logging
|
|
import os
|
|
import re
|
|
import json
|
|
|
|
import six
|
|
from six import text_type
|
|
import mozpack.path as mozpath
|
|
import taskgraph
|
|
from taskgraph.transforms.base import TransformSequence
|
|
from .. import GECKO
|
|
from taskgraph.util.docker import (
|
|
create_context_tar,
|
|
generate_context_hash,
|
|
image_path,
|
|
)
|
|
from taskgraph.util.schema import (
|
|
Schema,
|
|
)
|
|
from voluptuous import (
|
|
Optional,
|
|
Required,
|
|
)
|
|
from .task import task_description_schema
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
CONTEXTS_DIR = 'docker-contexts'
|
|
|
|
DIGEST_RE = re.compile('^[0-9a-f]{64}$')
|
|
|
|
IMAGE_BUILDER_IMAGE = (
|
|
'mozillareleases/image_builder:5.0.0'
|
|
"@sha256:"
|
|
"e510a9a9b80385f71c112d61b2f2053da625aff2b6d430411ac42e424c58953f"
|
|
)
|
|
|
|
transforms = TransformSequence()
|
|
|
|
docker_image_schema = Schema({
|
|
# Name of the docker image.
|
|
Required('name'): text_type,
|
|
|
|
# Name of the parent docker image.
|
|
Optional('parent'): text_type,
|
|
|
|
# Treeherder symbol.
|
|
Required('symbol'): text_type,
|
|
|
|
# relative path (from config.path) to the file the docker image was defined
|
|
# in.
|
|
Optional('job-from'): text_type,
|
|
|
|
# Arguments to use for the Dockerfile.
|
|
Optional('args'): {text_type: text_type},
|
|
|
|
# Name of the docker image definition under taskcluster/docker, when
|
|
# different from the docker image name.
|
|
Optional('definition'): text_type,
|
|
|
|
# List of package tasks this docker image depends on.
|
|
Optional('packages'): [text_type],
|
|
|
|
Optional(
|
|
"index",
|
|
description="information for indexing this build so its artifacts can be discovered",
|
|
): task_description_schema['index'],
|
|
|
|
Optional(
|
|
"cache",
|
|
description="Whether this image should be cached based on inputs.",
|
|
): bool,
|
|
})
|
|
|
|
|
|
transforms.add_validate(docker_image_schema)
|
|
|
|
|
|
@transforms.add
|
|
def fill_template(config, tasks):
|
|
if not taskgraph.fast and config.write_artifacts:
|
|
if not os.path.isdir(CONTEXTS_DIR):
|
|
os.makedirs(CONTEXTS_DIR)
|
|
|
|
for task in tasks:
|
|
image_name = task.pop('name')
|
|
job_symbol = task.pop('symbol')
|
|
args = task.pop('args', {})
|
|
packages = task.pop('packages', [])
|
|
parent = task.pop('parent', None)
|
|
|
|
for p in packages:
|
|
if "packages-{}".format(p) not in config.kind_dependencies_tasks:
|
|
raise Exception('Missing package job for {}-{}: {}'.format(
|
|
config.kind, image_name, p))
|
|
|
|
if not taskgraph.fast:
|
|
context_path = mozpath.relpath(image_path(image_name), GECKO)
|
|
if config.write_artifacts:
|
|
context_file = os.path.join(CONTEXTS_DIR, '{}.tar.gz'.format(image_name))
|
|
logger.info("Writing {} for docker image {}".format(context_file, image_name))
|
|
context_hash = create_context_tar(
|
|
GECKO, context_path,
|
|
context_file,
|
|
image_name,
|
|
args)
|
|
else:
|
|
context_hash = generate_context_hash(
|
|
GECKO, context_path,
|
|
image_name,
|
|
args)
|
|
else:
|
|
if config.write_artifacts:
|
|
raise Exception("Can't write artifacts if `taskgraph.fast` is set.")
|
|
context_hash = '0'*40
|
|
digest_data = [context_hash]
|
|
digest_data += [json.dumps(args, sort_keys=True)]
|
|
|
|
description = 'Build the docker image {} for use by dependent tasks'.format(
|
|
image_name)
|
|
|
|
args['DOCKER_IMAGE_PACKAGES'] = ' '.join('<{}>'.format(p)
|
|
for p in packages)
|
|
|
|
# Adjust the zstandard compression level based on the execution level.
|
|
# We use faster compression for level 1 because we care more about
|
|
# end-to-end times. We use slower/better compression for other levels
|
|
# because images are read more often and it is worth the trade-off to
|
|
# burn more CPU once to reduce image size.
|
|
zstd_level = '3' if int(config.params['level']) == 1 else '10'
|
|
|
|
# include some information that is useful in reconstructing this task
|
|
# from JSON
|
|
taskdesc = {
|
|
'label': '{}-{}'.format(config.kind, image_name),
|
|
'description': description,
|
|
'attributes': {
|
|
'image_name': image_name,
|
|
'artifact_prefix': 'public',
|
|
},
|
|
'expires-after': '28 days' if config.params.is_try() else '1 year',
|
|
'scopes': [],
|
|
'treeherder': {
|
|
'symbol': job_symbol,
|
|
'platform': 'taskcluster-images/opt',
|
|
'kind': 'other',
|
|
'tier': 1,
|
|
},
|
|
'run-on-projects': [],
|
|
'worker-type': 'images',
|
|
'worker': {
|
|
'implementation': 'docker-worker',
|
|
'os': 'linux',
|
|
'artifacts': [{
|
|
'type': 'file',
|
|
'path': '/workspace/image.tar.zst',
|
|
'name': 'public/image.tar.zst',
|
|
}],
|
|
'env': {
|
|
'CONTEXT_TASK_ID': {'task-reference': "<decision>"},
|
|
'CONTEXT_PATH': "public/docker-contexts/{}.tar.gz".format(image_name),
|
|
'HASH': context_hash,
|
|
'PROJECT': config.params['project'],
|
|
'IMAGE_NAME': image_name,
|
|
'DOCKER_IMAGE_ZSTD_LEVEL': zstd_level,
|
|
'DOCKER_BUILD_ARGS': {'task-reference': six.ensure_text(json.dumps(args))},
|
|
'GECKO_BASE_REPOSITORY': config.params['base_repository'],
|
|
'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
|
|
'GECKO_HEAD_REV': config.params['head_rev'],
|
|
},
|
|
'chain-of-trust': True,
|
|
'max-run-time': 7200,
|
|
# FIXME: We aren't currently propagating the exit code
|
|
},
|
|
}
|
|
# Retry for 'funsize-update-generator' if exit status code is -1
|
|
if image_name in ['funsize-update-generator']:
|
|
taskdesc['worker']['retry-exit-status'] = [-1]
|
|
|
|
worker = taskdesc['worker']
|
|
|
|
if image_name == 'image_builder':
|
|
worker['docker-image'] = IMAGE_BUILDER_IMAGE
|
|
digest_data.append("image-builder-image:{}".format(IMAGE_BUILDER_IMAGE))
|
|
else:
|
|
worker['docker-image'] = {'in-tree': 'image_builder'}
|
|
deps = taskdesc.setdefault('dependencies', {})
|
|
deps['docker-image'] = '{}-image_builder'.format(config.kind)
|
|
|
|
if packages:
|
|
deps = taskdesc.setdefault('dependencies', {})
|
|
for p in sorted(packages):
|
|
deps[p] = 'packages-{}'.format(p)
|
|
|
|
if parent:
|
|
deps = taskdesc.setdefault('dependencies', {})
|
|
deps['parent'] = '{}-{}'.format(config.kind, parent)
|
|
worker['env']['PARENT_TASK_ID'] = {
|
|
'task-reference': '<parent>',
|
|
}
|
|
if 'index' in task:
|
|
taskdesc['index'] = task['index']
|
|
|
|
if task.get('cache', True) and not taskgraph.fast:
|
|
taskdesc['cache'] = {
|
|
'type': 'docker-images.v2',
|
|
'name': image_name,
|
|
'digest-data': digest_data,
|
|
}
|
|
|
|
yield taskdesc
|