Bug 1245953 - Support for only running tasks when certain files change; r=garndt

Firefox's automation currently tends to run all the jobs all the time.
It is wasteful to do this. For example, running ESLint when the commit
only changes a .cpp file adds no value.

This commit adds support for only running tasks when certain files
change. The new-style tasks introduced by the previous commit have been
taught a "when" dictionary property that defines conditions that should
hold for the task to be executed. We define a "file_patterns" list that
defines lists of mozpack path matching expressions that will be matched
against the set of files changed by the changesets relevant to the
changeset being built. The eslint task has been updated to only run if
files related to it change.

Because conditions may not be accurate, we add a CLI argument to ignore
conditions and force all would-be-filtered tasks to run.

MozReview-Commit-ID: 3OeBSKAQAeg

testing/taskcluster/docs/index.rst

@@ -171,6 +171,31 @@ additional-parameters
    *optional* Dictionary of additional parameters to pass to template
    expansion.
 
+when
+   *optional* Dictionary of conditions that must be met for this task
+   to run. See the section below for more details.
+
+Conditional Execution
+---------------------
+
+The ``when`` generic task dictionary entry can declare conditions that
+must be true for a task to run. Valid entries in this dictionary are
+described below.
+
+file_patterns
+   List of path patterns that will be matched against all files changed.
+
+   The set of changed files is obtained from version control. If the changed
+   files could not be determined, this condition is ignored and no filtering
+   occurs.
+
+   Values use the ``mozpack`` matching code. ``*`` is a wildcard for
+   all path characters except ``/``. ``**`` matches all directories. To
+   e.g. match against all ``.js`` files, one would use ``**/*.js``.
+
+   If a single pattern matches a single changed file, the task will be
+   scheduled.
+
 Developing
 ==========
 

testing/taskcluster/mach_commands.py

@@ -274,9 +274,14 @@ class Graph(object):
     @CommandArgument('--dry-run',
         action='store_true', default=False,
         help="Stub out taskIds and date fields from the task definitions.")
+    @CommandArgument('--ignore-conditions',
+        action='store_true',
+        help='Run tasks even if their conditions are not met')
     def create_graph(self, **params):
         from functools import partial
 
+        from mozpack.path import match as mozpackmatch
+
         from slugid import nice as slugid
 
         import taskcluster_graph.transform.routes as routes_transform
@@ -324,6 +329,7 @@ class Graph(object):
         # Default to current time if querying the head rev fails
         pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
         vcs_info = query_vcs_info(params['head_repository'], params['head_rev'])
+        changed_files = set()
         if vcs_info:
             pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(vcs_info.pushdate))
 
@@ -333,6 +339,8 @@ class Graph(object):
                 sys.stderr.write('%s %s\n' % (
                     c['node'][0:12], c['desc'].splitlines()[0]))
 
+                changed_files |= set(c['files'])
+
         # Template parameters used when expanding the graph
         seen_images = {}
         parameters = dict(gaia_info().items() + {
@@ -390,6 +398,41 @@ class Graph(object):
             'name': 'task graph local'
         }
 
+        # Filter the job graph according to conditions met by this invocation run.
+        def should_run(task):
+            # Old style build or test task that doesn't define conditions. Always runs.
+            if 'when' not in task:
+                return True
+
+            # Command line override to not filter.
+            if params['ignore_conditions']:
+                return True
+
+            when = task['when']
+
+            # If the task defines file patterns and we have a set of changed
+            # files to compare against, only run if a file pattern matches one
+            # of the changed files.
+            file_patterns = when.get('file_patterns', None)
+            if file_patterns and changed_files:
+                for pattern in file_patterns:
+                    for path in changed_files:
+                        if mozpackmatch(path, pattern):
+                            sys.stderr.write('scheduling %s because pattern %s '
+                                             'matches %s\n' % (task['task'],
+                                                               pattern,
+                                                               path))
+                            return True
+
+                # No file patterns matched. Discard task.
+                sys.stderr.write('discarding %s because no relevant files changed\n' %
+                                 task['task'])
+                return False
+
+            return True
+
+        job_graph = filter(should_run, job_graph)
+
         all_routes = {}
 
         for build in job_graph:

testing/taskcluster/taskcluster_graph/commit_parser.py

@@ -350,6 +350,7 @@ def parse_commit(message, jobs):
             # TODO support declaring a different build type
             'build_type': name,
             'interactive': args.interactive,
+            'when': task.get('when', {})
         })
 
     # Times that test jobs will be scheduled

testing/taskcluster/tasks/branches/base_jobs.yml

@@ -309,4 +309,19 @@ tests:
 tasks:
   eslint-gecko:
     task: tasks/tests/eslint-gecko.yml
-    root: true
+    root: true
+    when:
+      file_patterns:
+        # Files that are likely audited.
+        - '**/*.js'
+        - '**/*.jsm'
+        - '**/*.jsx'
+        - '**/*.html'
+        - '**/*.xml'
+        # Run when eslint policies change.
+        - '**/.eslintignore'
+        - '**/*eslintrc*'
+        # The plugin implementing custom checks.
+        - 'testing/eslint-plugin-mozilla/**'
+        # Other misc lint related files.
+        - 'tools/lint/**'