Bug 1333255: use transforms to make docker image tasks, too; r=jonasfj

MozReview-Commit-ID: Eke3TjLbEfE

taskcluster/ci/docker-image/image.yml

@@ -1,68 +0,0 @@
----
-task:
-  created:
-    relative-datestamp: "0 seconds"
-  deadline:
-    relative-datestamp: "24 hours"
-  metadata:
-    name: 'Docker Image Build: {{image_name}}'
-    description: 'Build the docker image {{image_name}} for use by dependent tasks'
-    source: '{{source}}'
-    owner: mozilla-taskcluster-maintenance@mozilla.com
-  tags:
-    createdForUser: '{{owner}}'
-
-  workerType: gecko-images
-  provisionerId: aws-provisioner-v1
-  schedulerId: task-graph-scheduler
-
-  routes:
-      # Indexing routes to avoid building the same image twice
-      - index.{{index_image_prefix}}.level-{{level}}.{{image_name}}.latest
-      - index.{{index_image_prefix}}.level-{{level}}.{{image_name}}.pushdate.{{year}}.{{month}}-{{day}}-{{pushtime}}
-      - index.{{index_image_prefix}}.level-{{level}}.{{image_name}}.hash.{{context_hash}}
-      # Treeherder routes
-      - tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
-      - tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}
-
-  scopes:
-      - secrets:get:project/taskcluster/gecko/hgfingerprint
-      - docker-worker:cache:level-{{level}}-imagebuilder-v1
-
-  payload:
-    env:
-      HASH: '{{context_hash}}'
-      PROJECT: '{{project}}'
-      CONTEXT_URL: '{{context_url}}'
-      IMAGE_NAME: '{{image_name}}'
-      GECKO_BASE_REPOSITORY: '{{base_repository}}'
-      GECKO_HEAD_REPOSITORY: '{{head_repository}}'
-      GECKO_HEAD_REV: '{{head_rev}}'
-      HG_STORE_PATH: '/home/worker/checkouts/hg-store'
-    cache:
-      'level-{{level}}-imagebuilder-v1': '/home/worker/checkouts'
-    features:
-      dind: true
-      chainOfTrust: true
-      taskclusterProxy: true
-    image: '{{#docker_image}}image_builder{{/docker_image}}'
-    maxRunTime: 3600
-    artifacts:
-      '{{artifact_path}}':
-        type: 'file'
-        path: '/home/worker/workspace/artifacts/image.tar.zst'
-        expires:
-          relative-datestamp: "1 year"
-  extra:
-    imageMeta: # Useful when converting back from JSON in action tasks
-      level: '{{level}}'
-      contextHash: '{{context_hash}}'
-      imageName: '{{image_name}}'
-    treeherderEnv:
-      - staging
-      - production
-    treeherder:
-      jobKind: other
-      build:
-        platform: 'taskcluster-images'
-      groupSymbol: 'I'

taskcluster/ci/docker-image/kind.yml

@@ -3,17 +3,26 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.task.docker_image:load_tasks
-images_path: '../../../taskcluster/docker'
+
+transforms:
+  - taskgraph.transforms.docker_image:transforms
+  - taskgraph.transforms.task:transforms
 
 # make a task for each docker-image we might want.  For the moment, since we
 # write artifacts for each, these are whitelisted, but ideally that will change
 # (to use subdirectory clones of the proper directory), at which point we can
 # generate tasks for every docker image in the directory, secure in the
 # knowledge that unnecessary images will be omitted from the target task graph
-images:
+jobs:
-  desktop-test: dt
+  desktop-test:
-  desktop1604-test: dt16t
+    symbol: I(dt)
-  desktop-build: db
+  desktop1604-test:
-  tester: tst
+    symbol: I(dt16t)
-  lint: lnt
+  desktop-build:
-  android-gradle-build: agb
+    symbol: I(db)
+  tester:
+    symbol: I(tst)
+  lint:
+    symbol: I(lnt)
+  android-gradle-build:
+    symbol: I(agb)

taskcluster/taskgraph/task/docker_image.py

@@ -8,75 +8,38 @@ import logging
 import os
 import urllib2
 
-from . import base
+from . import transform
-from .. import GECKO
+from taskgraph.util.docker import INDEX_PREFIX
-from taskgraph.util.docker import (
+from taskgraph.transforms.base import TransformSequence, TransformConfig
-    docker_image,
-    generate_context_hash,
-    INDEX_PREFIX,
-)
 from taskgraph.util.taskcluster import get_artifact_url
-from taskgraph.util.templates import Templates
+from taskgraph.util.python_path import find_object
 
 logger = logging.getLogger(__name__)
 
 
-def load_tasks(kind, path, config, params, loaded_tasks):
+def transform_inputs(inputs, kind, path, config, params, loaded_tasks):
-    parameters = {
+    """
-        'pushlog_id': params.get('pushlog_id', 0),
+    Transform a sequence of inputs according to the transform configuration.
-        'pushdate': params['moz_build_date'],
+    """
-        'pushtime': params['moz_build_date'][8:],
+    transforms = TransformSequence()
-        'year': params['moz_build_date'][0:4],
+    for xform_path in config['transforms']:
-        'month': params['moz_build_date'][4:6],
+        transform = find_object(xform_path)
-        'day': params['moz_build_date'][6:8],
+        transforms.add(transform)
-        'project': params['project'],
-        'docker_image': docker_image,
-        'base_repository': params['base_repository'] or params['head_repository'],
-        'head_repository': params['head_repository'],
-        'head_ref': params['head_ref'] or params['head_rev'],
-        'head_rev': params['head_rev'],
-        'owner': params['owner'],
-        'level': params['level'],
-        'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml'
-                  .format(repo=params['head_repository'], rev=params['head_rev']),
-        'index_image_prefix': INDEX_PREFIX,
-        'artifact_path': 'public/image.tar.zst',
-    }
-
-    tasks = []
-    templates = Templates(path)
-    for image_name, image_symbol in config['images'].iteritems():
-        context_path = os.path.join('taskcluster', 'docker', image_name)
-        context_hash = generate_context_hash(GECKO, context_path, image_name)
-
-        image_parameters = dict(parameters)
-        image_parameters['image_name'] = image_name
-        image_parameters['context_hash'] = context_hash
-
-        image_task = templates.load('image.yml', image_parameters)
-        attributes = {'image_name': image_name}
-
-        # unique symbol for different docker image
-        if 'extra' in image_task['task']:
-            image_task['task']['extra']['treeherder']['symbol'] = image_symbol
-
-        # As an optimization, if the context hash exists for a high level, that image
-        # task ID will be used.  The reasoning behind this is that eventually everything ends
-        # up on level 3 at some point if most tasks use this as a common image
-        # for a given context hash, a worker within Taskcluster does not need to contain
-        # the same image per branch.
-        index_paths = ['{}.level-{}.{}.hash.{}'.format(
-            INDEX_PREFIX, level, image_name, context_hash)
-            for level in reversed(range(int(params['level']), 4))]
-
-        tasks.append(DockerImageTask(kind, 'build-docker-image-' + image_name,
-                                     task=image_task['task'], attributes=attributes,
-                                     index_paths=index_paths))
 
+    # perform the transformations
+    trans_config = TransformConfig(kind, path, config, params)
+    tasks = [DockerImageTask(kind, t)
+             for t in transforms(trans_config, inputs)]
     return tasks
 
 
-class DockerImageTask(base.Task):
+def load_tasks(kind, path, config, params, loaded_tasks):
+    return transform_inputs(
+        transform.get_inputs(kind, path, config, params, loaded_tasks),
+        kind, path, config, params, loaded_tasks)
+
+
+class DockerImageTask(transform.TransformTask):
     def get_dependencies(self, taskgraph):
         return []
 
@@ -108,9 +71,6 @@ class DockerImageTask(base.Task):
         index_paths = ['{}.level-{}.{}.hash.{}'.format(
             INDEX_PREFIX, level, image_name, context_hash)
             for level in reversed(range(int(imgMeta['level']), 4))]
-        docker_image_task = cls(kind='docker-image',
+        task_dict['index_paths'] = index_paths
-                                label=task_dict['label'],
+        docker_image_task = cls(kind='docker-image', task=task_dict)
-                                attributes=task_dict['attributes'],
-                                task=task_dict['task'],
-                                index_paths=index_paths)
         return docker_image_task

taskcluster/taskgraph/task/transform.py

@@ -84,8 +84,8 @@ class TransformTask(base.Task):
     """
 
     def __init__(self, kind, task):
-        self.dependencies = task['dependencies']
+        self.dependencies = task.get('dependencies', {})
-        self.when = task['when']
+        self.when = task.get('when', {})
         super(TransformTask, self).__init__(kind, task['label'],
                                             task['attributes'], task['task'],
                                             index_paths=task.get('index-paths'))

taskcluster/taskgraph/test/test_task_docker_image.py

@@ -1,35 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import unittest
-import os
-
-from ..task import docker_image
-from mozunit import main
-
-
-KIND_PATH = os.path.join(docker_image.GECKO, 'taskcluster', 'ci', 'docker-image')
-
-
-class TestDockerImageKind(unittest.TestCase):
-
-    def setUp(self):
-        self.task = docker_image.DockerImageTask(
-            'docker-image',
-            KIND_PATH,
-            {},
-            {},
-            index_paths=[])
-
-    def test_get_task_dependencies(self):
-        # this one's easy!
-        self.assertEqual(self.task.get_dependencies(None), [])
-
-    # TODO: optimize_task
-
-
-if __name__ == '__main__':
-    main()

taskcluster/taskgraph/test/test_taskgraph.py

@@ -14,7 +14,7 @@ from mozunit import main
 from taskgraph.util.docker import INDEX_PREFIX
 
 
-class TestTargetTasks(unittest.TestCase):
+class TestTaskGraph(unittest.TestCase):
 
     def test_from_json(self):
         task = {
@@ -40,10 +40,12 @@ class TestTargetTasks(unittest.TestCase):
                     'task': {'task': 'def'},
                 }),
             'b': DockerImageTask(kind='docker-image',
-                                 label='b',
+                                 task={
-                                 attributes={},
+                                     'label': 'b',
-                                 task=task,
+                                     'attributes': {},
-                                 index_paths=index_paths),
+                                     'task': task,
+                                     'index_paths': index_paths,
+                                 }),
         }, graph=Graph(nodes={'a', 'b'}, edges=set()))
 
         tasks, new_graph = TaskGraph.from_json(graph.to_json())

taskcluster/taskgraph/transforms/docker_image.py

@@ -0,0 +1,126 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the upload-symbols task description template,
+  taskcluster/ci/upload-symbols/job-template.yml
+into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+
+from taskgraph.transforms.base import TransformSequence
+from .. import GECKO
+from taskgraph.util.docker import (
+    docker_image,
+    generate_context_hash,
+    INDEX_PREFIX,
+)
+
+transforms = TransformSequence()
+
+ROUTE_TEMPLATES = [
+    'index.{index_prefix}.level-{level}.{image_name}.latest',
+    'index.{index_prefix}.level-{level}.{image_name}.pushdate.{year}.{month}-{day}-{pushtime}',
+    'index.{index_prefix}.level-{level}.{image_name}.hash.{context_hash}',
+]
+
+
+@transforms.add
+def fill_template(config, tasks):
+    for task in tasks:
+        image_name = task.pop('name')
+        job_symbol = task.pop('symbol')
+
+        context_path = os.path.join('taskcluster', 'docker', image_name)
+        context_hash = generate_context_hash(GECKO, context_path, image_name)
+
+        description = 'Build the docker image {} for use by dependent tasks'.format(
+            image_name)
+
+        routes = []
+        for tpl in ROUTE_TEMPLATES:
+            routes.append(tpl.format(
+                index_prefix=INDEX_PREFIX,
+                level=config.params['level'],
+                image_name=image_name,
+                project=config.params['project'],
+                head_rev=config.params['head_rev'],
+                pushlog_id=config.params.get('pushlog_id', 0),
+                pushtime=config.params['moz_build_date'][8:],
+                year=config.params['moz_build_date'][0:4],
+                month=config.params['moz_build_date'][4:6],
+                day=config.params['moz_build_date'][6:8],
+                context_hash=context_hash,
+            ))
+
+        # As an optimization, if the context hash exists for a high level, that image
+        # task ID will be used.  The reasoning behind this is that eventually everything ends
+        # up on level 3 at some point if most tasks use this as a common image
+        # for a given context hash, a worker within Taskcluster does not need to contain
+        # the same image per branch.
+        index_paths = ['{}.level-{}.{}.hash.{}'.format(
+            INDEX_PREFIX, level, image_name, context_hash)
+            for level in reversed(range(int(config.params['level']), 4))]
+
+        # include some information that is useful in reconstructing this task
+        # from JSON
+        extra = {
+            'imageMeta': {
+                'level': config.params['level'],
+                'contextHash': context_hash,
+                'imageName': image_name,
+            },
+        }
+
+        taskdesc = {
+            'label': 'build-docker-image-' + image_name,
+            'description': description,
+            'attributes': {'image_name': image_name},
+            'expires-after': '1 year',
+            'routes': routes,
+            'index-paths': index_paths,
+            'scopes': ['secrets:get:project/taskcluster/gecko/hgfingerprint'],
+            'extra': extra,
+            'treeherder': {
+                'symbol': job_symbol,
+                'platform': 'taskcluster-images/opt',
+                'kind': 'other',
+                'tier': 1,
+            },
+            'run-on-projects': [],
+            'worker-type': 'aws-provisioner-v1/gecko-images',
+            # can't use {in-tree: ..} here, otherwise we might try to build
+            # this image..
+            'worker': {
+                'implementation': 'docker-worker',
+                'docker-image': docker_image('image_builder'),
+                'caches': [{
+                    'type': 'persistent',
+                    'name': 'level-{}-imagebuilder-v1'.format(config.params['level']),
+                    'mount-point': '/home/worker/checkouts',
+                }],
+                'artifacts': [{
+                    'type': 'file',
+                    'path': '/home/worker/workspace/artifacts/image.tar.zst',
+                    'name': 'public/image.tar.zst',
+                }],
+                'env': {
+                    'HG_STORE_PATH': '/home/worker/checkouts/hg-store',
+                    'HASH': context_hash,
+                    'PROJECT': config.params['project'],
+                    'IMAGE_NAME': image_name,
+                    'GECKO_BASE_REPOSITORY': config.params['base_repository'],
+                    'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
+                    'GECKO_HEAD_REV': config.params['head_rev'],
+                },
+                'chain-of-trust': True,
+                'docker-in-docker': True,
+                'taskcluster-proxy': True,
+                'max-run-time': 3600,
+            },
+        }
+
+        yield taskdesc

taskcluster/taskgraph/transforms/task.py

@@ -162,6 +162,7 @@ task_description_schema = Schema({
         Required('allow-ptrace', default=False): bool,
         Required('loopback-video', default=False): bool,
         Required('loopback-audio', default=False): bool,
+        Required('docker-in-docker', default=False): bool,  # (aka 'dind')
 
         # caches to set up for the task
         Optional('caches'): [{
@@ -375,6 +376,7 @@ GROUP_NAMES = {
     'tc-BMcs': 'Beetmover checksums, executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
+    'I': 'Docker Image Builds',
     'TL': 'Toolchain builds for Linux 64-bits',
     'TM': 'Toolchain builds for OSX',
     'TW32': 'Toolchain builds for Windows 32-bits',
@@ -460,6 +462,9 @@ def build_docker_worker_payload(config, task, task_def):
     if worker.get('chain-of-trust'):
         features['chainOfTrust'] = True
 
+    if worker.get('docker-in-docker'):
+        features['dind'] = True
+
     if task.get('needs-sccache'):
         features['taskclusterProxy'] = True
         task_def['scopes'].append(
@@ -480,10 +485,11 @@ def build_docker_worker_payload(config, task, task_def):
             task_def['scopes'].append('docker-worker:capability:device:' + capitalized)
 
     task_def['payload'] = payload = {
-        'command': worker['command'],
         'image': image,
         'env': worker['env'],
     }
+    if 'command' in worker:
+        payload['command'] = worker['command']
 
     if 'max-run-time' in worker:
         payload['maxRunTime'] = worker['max-run-time']