Bug 1537574 - Use mozbuild's backend-out-of-date logic for RecursiveMake; r=firefox-build-system-reviewers,chmanchester

If mozbuild parsing fails due to a missing file (eg: a file not existing
in UNIFIED_SOURCES), then no Makefiles are written out, but
config.status exists. This would cause mozbuild to think that configure
doesn't need to run, and rely on make to perform the backend-out-of-date
check in rebuild-backend.mk. Unfortunately since no Makefiles were
written, the make command fails immediately and no attempt is made to
re-create the backend. Note that this is only a problem if the first
mozbuild parsing from a clobber build fails, otherwise there is
typically a top-level Makefile from a previous build to call into (at
which point make can determine it is out-of-date, and re-invoke itself).

The fix is to have the RecursiveMake backend re-use the same logic that
was introduced into mozbuild for alternate backends, and remove
rebuild-backend.mk. This way, mozbuild can always determine if the
backend needs to be regenerated, even if the initial parsing failed.

Test code was also relying on rebuild-backend.mk to generate the
TestBackend, but moving backend_out_of_date() into MozbuildObject allows
this code to be shared.

Differential Revision: https://phabricator.services.mozilla.com/D26262

Makefile.in

@@ -72,26 +72,6 @@ CLOBBER: $(topsrcdir)/CLOBBER
 	@exit 1
 endif
 
-# Regenerate the build backend if it is out of date. We only have this rule in
-# this main make file because having it in rules.mk and applied to partial tree
-# builds resulted in a world of hurt. Gory details are in bug 877308.
-#
-# The mach build driver will ensure the backend is up to date for partial tree
-# builds. This cleanly avoids most of the pain.
-
-ifndef TEST_MOZBUILD
-
-.PHONY: backend
-backend: $(BUILD_BACKEND_FILES)
-
-include $(topsrcdir)/build/rebuild-backend.mk
-
-Makefile: $(BUILD_BACKEND_FILES)
-	@$(TOUCH) $@
-
-default:: $(BUILD_BACKEND_FILES)
-endif
-
 install_manifests := \
   $(addprefix dist/,branding include public private xpi-stage) \
   _tests \

build/rebuild-backend.mk

@@ -1,31 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-BACKEND_GENERATION_SCRIPT ?= config.status
-
-# A traditional rule would look like this:
-#    backend.%:
-#        @echo do stuff
-#
-# But with -j<n>, and multiple items in BUILD_BACKEND_FILES, the command would
-# run multiple times in parallel.
-#
-# "Fortunately", make has some weird semantics for pattern rules: if there are
-# multiple targets in a pattern rule and each of them is matched at most once,
-# the command will only run once. So:
-#     backend%RecursiveMakeBackend backend%FasterMakeBackend:
-#         @echo do stuff
-#     backend: backend.RecursiveMakeBackend backend.FasterMakeBackend
-# would only execute the command once.
-#
-# Credit where due: http://stackoverflow.com/questions/2973445/gnu-makefile-rule-generating-a-few-targets-from-a-single-source-file/3077254#3077254
-$(subst .,%,$(BUILD_BACKEND_FILES)):
-	@echo 'Build configuration changed. Regenerating backend.'
-	$(PYTHON) $(BACKEND_GENERATION_SCRIPT)
-
-define build_backend_rule
-$(1): $$(wildcard $$(shell cat $(1).in))
-
-endef
-$(foreach file,$(BUILD_BACKEND_FILES),$(eval $(call build_backend_rule,$(file))))

python/mozbuild/mozbuild/base.py

@@ -197,6 +197,49 @@ class MozbuildObject(ProcessExecutionMixin):
 
         return mozpath.normsep(os.path.normpath(topobjdir))
 
+    def build_out_of_date(self, output, dep_file):
+        if not os.path.isfile(output):
+            print(" Output reference file not found: %s" % output)
+            return True
+        if not os.path.isfile(dep_file):
+            print(" Dependency file not found: %s" % dep_file)
+            return True
+
+        deps = []
+        with open(dep_file, 'r') as fh:
+            deps = fh.read().splitlines()
+
+        mtime = os.path.getmtime(output)
+        for f in deps:
+            try:
+                dep_mtime = os.path.getmtime(f)
+            except OSError as e:
+                if e.errno == errno.ENOENT:
+                    print(" Input not found: %s" % f)
+                    return True
+                raise
+            if dep_mtime > mtime:
+                print(" %s is out of date with respect to %s" % (output, f))
+                return True
+        return False
+
+    def backend_out_of_date(self, backend_file):
+        if not os.path.isfile(backend_file):
+            return True
+
+        # Check if any of our output files have been removed since
+        # we last built the backend, re-generate the backend if
+        # so.
+        outputs = []
+        with open(backend_file, 'r') as fh:
+            outputs = fh.read().splitlines()
+        for output in outputs:
+            if not os.path.isfile(mozpath.join(self.topobjdir, output)):
+                return True
+
+        dep_file = '%s.in' % backend_file
+        return self.build_out_of_date(backend_file, dep_file)
+
     @property
     def topobjdir(self):
         if self._topobjdir is None:

python/mozbuild/mozbuild/controller/building.py

@@ -1014,49 +1014,6 @@ class BuildDriver(MozbuildObject):
                 if directory.startswith('/'):
                     directory = directory[1:]
 
-            def build_out_of_date(output, dep_file):
-                if not os.path.isfile(output):
-                    print(" Output reference file not found: %s" % output)
-                    return True
-                if not os.path.isfile(dep_file):
-                    print(" Configure dependency file not found: %s" % dep_file)
-                    return True
-
-                deps = []
-                with open(dep_file, 'r') as fh:
-                    deps = fh.read().splitlines()
-
-                mtime = os.path.getmtime(output)
-                for f in deps:
-                    try:
-                        dep_mtime = os.path.getmtime(f)
-                    except OSError as e:
-                        if e.errno == errno.ENOENT:
-                            print(" Configure input not found: %s" % f)
-                            return True
-                        raise
-                    if dep_mtime > mtime:
-                        print(" %s is out of date with respect to %s" % (output, f))
-                        return True
-                return False
-
-            def backend_out_of_date(backend_file):
-                if not os.path.isfile(backend_file):
-                    return True
-
-                # Check if any of our output files have been removed since
-                # we last built the backend, re-generate the backend if
-                # so.
-                outputs = []
-                with open(backend_file, 'r') as fh:
-                    outputs = fh.read().splitlines()
-                for output in outputs:
-                    if not os.path.isfile(mozpath.join(self.topobjdir, output)):
-                        return True
-
-                dep_file = '%s.in' % backend_file
-                return build_out_of_date(backend_file, dep_file)
-
             monitor.start_resource_recording()
 
             self.mach_context.command_attrs['clobber'] = False
@@ -1084,7 +1041,7 @@ class BuildDriver(MozbuildObject):
             config_rc = None
             # Even if we have a config object, it may be out of date
             # if something that influences its result has changed.
-            if config is None or build_out_of_date(mozpath.join(self.topobjdir,
+            if config is None or self.build_out_of_date(mozpath.join(self.topobjdir,
                                                                      'config.status'),
                                                         mozpath.join(self.topobjdir,
                                                                      'config_status_deps.in')):
@@ -1106,9 +1063,8 @@ class BuildDriver(MozbuildObject):
 
             status = None
 
-            if 'Make' not in active_backend:
             if (not config_rc and
-                    backend_out_of_date(mozpath.join(self.topobjdir,
+                self.backend_out_of_date(mozpath.join(self.topobjdir,
                                                       'backend.%sBackend' %
                                                       active_backend))):
                 print('Build configuration changed. Regenerating backend.')
@@ -1116,6 +1072,7 @@ class BuildDriver(MozbuildObject):
                         mozpath.join(self.topobjdir, 'config.status')]
                 self.run_process(args, cwd=self.topobjdir, pass_thru=True)
 
+            if 'Make' not in active_backend:
                 # client.mk has its own handling of MOZ_PARALLEL_BUILD so the
                 # make backend can determine when to run in single-threaded mode
                 # or parallel mode. For other backends, we can pass in the value
@@ -1181,15 +1138,6 @@ class BuildDriver(MozbuildObject):
                              'instead of {target_pairs}.')
                     target_pairs = new_pairs
 
-                # Ensure build backend is up to date. The alternative is to
-                # have rules in the invoked Makefile to rebuild the build
-                # backend. But that involves make reinvoking itself and there
-                # are undesired side-effects of this. See bug 877308 for a
-                # comprehensive history lesson.
-                self._run_make(directory=self.topobjdir, target='backend',
-                    line_handler=output.on_line, log=False,
-                    print_directory=False, keep_going=keep_going)
-
                 # Build target pairs.
                 for make_dir, make_target in target_pairs:
                     # We don't display build status messages during partial

testing/mozbase/moztest/moztest/resolve.py

@@ -544,16 +544,12 @@ class TestResolver(MozbuildObject):
         # If installing tests is going to result in re-generating the build
         # backend, we need to do this here, so that the updated contents of
         # all-tests.pkl make it to the set of tests to run.
-        self._run_make(
+        if self.backend_out_of_date(mozpath.join(self.topobjdir,
-            target='backend.TestManifestBackend', pass_thru=True, print_directory=False,
+                                                 'backend.TestManifestBackend'
-            filename=mozpath.join(self.topsrcdir, 'build', 'rebuild-backend.mk'),
+                                                 )):
-            append_env={
+            print("Test configuration changed. Regenerating backend.")
-                b'PYTHON': self.virtualenv_manager.python_path,
+            from mozbuild.gen_test_backend import gen_test_backend
-                b'BUILD_BACKEND_FILES': b'backend.TestManifestBackend',
+            gen_test_backend()
-                b'BACKEND_GENERATION_SCRIPT': mozpath.join(
-                    self.topsrcdir, 'build', 'gen_test_backend.py'),
-            },
-        )
 
         self._tests = TestMetadata(os.path.join(self.topobjdir,
                                                 'all-tests.pkl'),