Bug 1395717 - Use filename source-test tasks are defined in as part of their label, r=dustin

This creates a new "job-from" field that contains the relative filename the job was defined
in. The filename is relative to 'config.path'. If the task came from the 'jobs' key defined
in kind.yml, this field will be set to 'kind.yml'.

MozReview-Commit-ID: 9e1tEb6XuZT

taskcluster/ci/source-test/doc.yml

@@ -1,4 +1,4 @@
-doc-generate:
+generate:
     description: Generate the Sphinx documentation
     platform: lint/opt
     treeherder:
@@ -25,7 +25,7 @@ doc-generate:
             - '**/*.rst'
             - 'tools/docs/**'
 
-doc-upload:
+upload:
     description: Generate and upload the Sphinx documentation
     platform: lint/opt
     treeherder:

taskcluster/ci/source-test/kind.yml

@@ -14,7 +14,7 @@ jobs-from:
    - doc.yml
    - mocha.yml
    - mozlint.yml
-   - python-tests.yml
+   - python.yml
    - webidl.yml
 
 # This is used by run-task based tasks to lookup which build task it

taskcluster/ci/source-test/mozlint.yml

@@ -1,4 +1,4 @@
-mozlint-eslint:
+eslint:
     description: JS lint check
     platform: lint/opt
     treeherder:
@@ -36,7 +36,27 @@ mozlint-eslint:
             - 'python/mozlint/**'
             - 'tools/lint/**'
 
-mozlint-py-flake8:
+py-compat:
+    description: lint for python 2/3 compatibility issues
+    platform: lint/opt
+    treeherder:
+        symbol: py-compat
+        kind: test
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
+    worker:
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: mach
+        mach: lint -l py2 -l py3 -f treeherder
+    when:
+        files-changed:
+            - '**/*.py'
+            - 'python/mozlint/**'
+            - 'tools/lint/**'
+
+py-flake8:
     description: flake8 run over the gecko codebase
     platform: lint/opt
     treeherder:
@@ -57,48 +77,6 @@ mozlint-py-flake8:
             - 'python/mozlint/**'
             - 'tools/lint/**'
 
-mozlint-yaml:
-    description: yamllint run over the gecko codebase
-    platform: lint/opt
-    treeherder:
-        symbol: yaml
-        kind: test
-        tier: 1
-    worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
-    worker:
-        docker-image: {in-tree: "lint"}
-        max-run-time: 1800
-    run:
-        using: mach
-        mach: lint -l yaml -f treeherder
-    when:
-        files-changed:
-            - '**/*.yml'
-            - '**/*.yaml'
-            - '**/.ymllint'
-            - 'python/mozlint/**'
-            - 'tools/lint/**'
-
-mozlint-py-compat:
-    description: lint for python 2/3 compatibility issues
-    platform: lint/opt
-    treeherder:
-        symbol: py-compat
-        kind: test
-        tier: 1
-    worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
-    worker:
-        docker-image: {in-tree: "lint"}
-        max-run-time: 1800
-    run:
-        using: mach
-        mach: lint -l py2 -l py3 -f treeherder
-    when:
-        files-changed:
-            - '**/*.py'
-            - 'python/mozlint/**'
-            - 'tools/lint/**'
-
 wptlint-gecko:
     description: web-platform-tests linter
     platform: lint/opt
@@ -121,3 +99,25 @@ wptlint-gecko:
             - 'testing/web-platform/mozilla/meta/MANIFEST.json'
             - 'python/mozlint/**'
             - 'tools/lint/**'
+
+yaml:
+    description: yamllint run over the gecko codebase
+    platform: lint/opt
+    treeherder:
+        symbol: yaml
+        kind: test
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
+    worker:
+        docker-image: {in-tree: "lint"}
+        max-run-time: 1800
+    run:
+        using: mach
+        mach: lint -l yaml -f treeherder
+    when:
+        files-changed:
+            - '**/*.yml'
+            - '**/*.yaml'
+            - '**/.ymllint'
+            - 'python/mozlint/**'
+            - 'tools/lint/**'

taskcluster/ci/source-test/webidl.yml

@@ -1,4 +1,4 @@
-webidl-test:
+test:
     description: WebIDL parser tests
     platform: lint/opt
     treeherder:

taskcluster/taskgraph/loader/transform.py

@@ -5,7 +5,6 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
-import itertools
 
 from ..util.templates import merge
 from ..util.yaml import load_yaml
@@ -33,13 +32,17 @@ def loader(kind, path, config, params, loaded_tasks):
     """
     def jobs():
         defaults = config.get('job-defaults')
-        jobs = config.get('jobs', {}).iteritems()
+        for name, job in config.get('jobs', {}).iteritems():
-        jobs_from = itertools.chain.from_iterable(
-            load_yaml(path, filename).iteritems()
-            for filename in config.get('jobs-from', {}))
-        for name, job in itertools.chain(jobs, jobs_from):
             if defaults:
                 job = merge(defaults, job)
+            job['job-from'] = 'kind.yml'
+            yield name, job
+
+        for filename in config.get('jobs-from', []):
+            for name, job in load_yaml(path, filename).iteritems():
+                if defaults:
+                    job = merge(defaults, job)
+                job['job-from'] = filename
                 yield name, job
 
     for name, job in jobs():

taskcluster/taskgraph/transforms/job/__init__.py

@@ -48,6 +48,7 @@ job_description_schema = Schema({
     # taskcluster/taskgraph/transforms/task.py for the schema details.
     Required('description'): task_description_schema['description'],
     Optional('attributes'): task_description_schema['attributes'],
+    Optional('job-from'): task_description_schema['job-from'],
     Optional('dependencies'): task_description_schema['dependencies'],
     Optional('expires-after'): task_description_schema['expires-after'],
     Optional('routes'): task_description_schema['routes'],

taskcluster/taskgraph/transforms/push_apk.py

@@ -27,6 +27,7 @@ push_apk_description_schema = Schema({
     Required('name'): basestring,
     Required('label'): basestring,
     Required('description'): basestring,
+    Required('job-from'): basestring,
     Required('attributes'): object,
     Required('treeherder'): object,
     Required('run-on-projects'): list,

taskcluster/taskgraph/transforms/push_apk_breakpoint.py

@@ -25,6 +25,7 @@ push_apk_breakpoint_description_schema = Schema({
     Required('name'): basestring,
     Required('label'): basestring,
     Required('description'): basestring,
+    Required('job-from'): basestring,
     Required('attributes'): object,
     Required('worker-type'): None,
     Required('worker'): object,

taskcluster/taskgraph/transforms/source_test.py

@@ -10,6 +10,7 @@ treeherder configuration and attributes for that platform.
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
+import os
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.job import job_description_schema
@@ -68,9 +69,13 @@ def validate(config, jobs):
 
 
 @transforms.add
-def set_job_try_name(config, jobs):
+def set_job_name(config, jobs):
     for job in jobs:
         job.setdefault('attributes', {}).setdefault('job_try_name', job['name'])
+
+        if 'job-from' in job and job['job-from'] != 'kind.yml':
+            from_name = os.path.splitext(job['job-from'])[0]
+            job['name'] = '{}-{}'.format(from_name, job['name'])
         yield job
 
 

taskcluster/taskgraph/transforms/task.py

@@ -55,6 +55,9 @@ task_description_schema = Schema({
     # attributes for this task
     Optional('attributes'): {basestring: object},
 
+    # relative path (from config.path) to the file task was defined in
+    Optional('job-from'): basestring,
+
     # dependencies of this task, keyed by name; these are passed through
     # verbatim and subject to the interpretation of the Task's get_dependencies
     # method.