Bug 1245953 - Support for only running tasks when certain files change; r=garndt

Firefox's automation currently tends to run all the jobs all the time.
It is wasteful to do this. For example, running ESLint when the commit
only changes a .cpp file adds no value.

This commit adds support for only running tasks when certain files
change. The new-style tasks introduced by the previous commit have been
taught a "when" dictionary property that defines conditions that should
hold for the task to be executed. We define a "file_patterns" list that
defines lists of mozpack path matching expressions that will be matched
against the set of files changed by the changesets relevant to the
changeset being built. The eslint task has been updated to only run if
files related to it change.

Because conditions may not be accurate, we add a CLI argument to ignore
conditions and force all would-be-filtered tasks to run.

MozReview-Commit-ID: 3OeBSKAQAeg

testing/taskcluster/mach_commands.py

@@ -274,9 +274,14 @@ class Graph(object):
     @CommandArgument('--dry-run',
         action='store_true', default=False,
         help="Stub out taskIds and date fields from the task definitions.")
+    @CommandArgument('--ignore-conditions',
+        action='store_true',
+        help='Run tasks even if their conditions are not met')
     def create_graph(self, **params):
         from functools import partial
 
+        from mozpack.path import match as mozpackmatch
+
         from slugid import nice as slugid
 
         import taskcluster_graph.transform.routes as routes_transform
@@ -324,6 +329,7 @@ class Graph(object):
         # Default to current time if querying the head rev fails
         pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
         vcs_info = query_vcs_info(params['head_repository'], params['head_rev'])
+        changed_files = set()
         if vcs_info:
             pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(vcs_info.pushdate))
 
@@ -333,6 +339,8 @@ class Graph(object):
                 sys.stderr.write('%s %s\n' % (
                     c['node'][0:12], c['desc'].splitlines()[0]))
 
+                changed_files |= set(c['files'])
+
         # Template parameters used when expanding the graph
         seen_images = {}
         parameters = dict(gaia_info().items() + {
@@ -390,6 +398,41 @@ class Graph(object):
             'name': 'task graph local'
         }
 
+        # Filter the job graph according to conditions met by this invocation run.
+        def should_run(task):
+            # Old style build or test task that doesn't define conditions. Always runs.
+            if 'when' not in task:
+                return True
+
+            # Command line override to not filter.
+            if params['ignore_conditions']:
+                return True
+
+            when = task['when']
+
+            # If the task defines file patterns and we have a set of changed
+            # files to compare against, only run if a file pattern matches one
+            # of the changed files.
+            file_patterns = when.get('file_patterns', None)
+            if file_patterns and changed_files:
+                for pattern in file_patterns:
+                    for path in changed_files:
+                        if mozpackmatch(path, pattern):
+                            sys.stderr.write('scheduling %s because pattern %s '
+                                             'matches %s\n' % (task['task'],
+                                                               pattern,
+                                                               path))
+                            return True
+
+                # No file patterns matched. Discard task.
+                sys.stderr.write('discarding %s because no relevant files changed\n' %
+                                 task['task'])
+                return False
+
+            return True
+
+        job_graph = filter(should_run, job_graph)
+
         all_routes = {}
 
         for build in job_graph: