corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1537574 - Use mozbuild's backend-out-of-date logic for RecursiveMake; r=firefox-build-system-reviewers,chmanchester

Browse Source 
If mozbuild parsing fails due to a missing file (eg: a file not existing
in UNIFIED_SOURCES), then no Makefiles are written out, but
config.status exists. This would cause mozbuild to think that configure
doesn't need to run, and rely on make to perform the backend-out-of-date
check in rebuild-backend.mk. Unfortunately since no Makefiles were
written, the make command fails immediately and no attempt is made to
re-create the backend. Note that this is only a problem if the first
mozbuild parsing from a clobber build fails, otherwise there is
typically a top-level Makefile from a previous build to call into (at
which point make can determine it is out-of-date, and re-invoke itself).

The fix is to have the RecursiveMake backend re-use the same logic that
was introduced into mozbuild for alternate backends, and remove
rebuild-backend.mk. This way, mozbuild can always determine if the
backend needs to be regenerated, even if the initial parsing failed.

Test code was also relying on rebuild-backend.mk to generate the
TestBackend, but moving backend_out_of_date() into MozbuildObject allows
this code to be shared.

Differential Revision: https://phabricator.services.mozilla.com/D26262
This commit is contained in:
Mike Shal
2019-04-10 22:14:27 +00:00
parent 4b50db52a5
commit c04e18fa25
6 changed files with 62 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
python/mozbuild/mozbuild/base.py
View File

@@ -197,6 +197,49 @@ class MozbuildObject(ProcessExecutionMixin):
return mozpath.normsep(os.path.normpath(topobjdir))
def build_out_of_date(self, output, dep_file):
if not os.path.isfile(output):
print(" Output reference file not found: %s" % output)
return True
if not os.path.isfile(dep_file):
print(" Dependency file not found: %s" % dep_file)
return True
deps = []
with open(dep_file, 'r') as fh:
deps = fh.read().splitlines()
mtime = os.path.getmtime(output)
for f in deps:
try:
dep_mtime = os.path.getmtime(f)
except OSError as e:
if e.errno == errno.ENOENT:
print(" Input not found: %s" % f)
return True
raise
if dep_mtime > mtime:
print(" %s is out of date with respect to %s" % (output, f))
return True
return False
def backend_out_of_date(self, backend_file):
if not os.path.isfile(backend_file):
return True
# Check if any of our output files have been removed since
# we last built the backend, re-generate the backend if
# so.
outputs = []
with open(backend_file, 'r') as fh:
outputs = fh.read().splitlines()
for output in outputs:
if not os.path.isfile(mozpath.join(self.topobjdir, output)):
return True
dep_file = '%s.in' % backend_file
return self.build_out_of_date(backend_file, dep_file)
@property
def topobjdir(self):
if self._topobjdir is None:

78
python/mozbuild/mozbuild/controller/building.py
View File

@@ -1014,49 +1014,6 @@ class BuildDriver(MozbuildObject):
if directory.startswith('/'):
directory = directory[1:]
def build_out_of_date(output, dep_file):
if not os.path.isfile(output):
print(" Output reference file not found: %s" % output)
return True
if not os.path.isfile(dep_file):
print(" Configure dependency file not found: %s" % dep_file)
return True
deps = []
with open(dep_file, 'r') as fh:
deps = fh.read().splitlines()
mtime = os.path.getmtime(output)
for f in deps:
try:
dep_mtime = os.path.getmtime(f)
except OSError as e:
if e.errno == errno.ENOENT:
print(" Configure input not found: %s" % f)
return True
raise
if dep_mtime > mtime:
print(" %s is out of date with respect to %s" % (output, f))
return True
return False
def backend_out_of_date(backend_file):
if not os.path.isfile(backend_file):
return True
# Check if any of our output files have been removed since
# we last built the backend, re-generate the backend if
# so.
outputs = []
with open(backend_file, 'r') as fh:
outputs = fh.read().splitlines()
for output in outputs:
if not os.path.isfile(mozpath.join(self.topobjdir, output)):
return True
dep_file = '%s.in' % backend_file
return build_out_of_date(backend_file, dep_file)
monitor.start_resource_recording()
self.mach_context.command_attrs['clobber'] = False
@@ -1084,10 +1041,10 @@ class BuildDriver(MozbuildObject):
config_rc = None
# Even if we have a config object, it may be out of date
# if something that influences its result has changed.
if config is None or build_out_of_date(mozpath.join(self.topobjdir,
'config.status'),
mozpath.join(self.topobjdir,
'config_status_deps.in')):
if config is None or self.build_out_of_date(mozpath.join(self.topobjdir,
'config.status'),
mozpath.join(self.topobjdir,
'config_status_deps.in')):
if previous_backend and 'Make' not in previous_backend:
clobber_requested = self._clobber_configure()
@@ -1106,16 +1063,16 @@ class BuildDriver(MozbuildObject):
status = None
if 'Make' not in active_backend:
if (not config_rc and
backend_out_of_date(mozpath.join(self.topobjdir,
'backend.%sBackend' %
active_backend))):
print('Build configuration changed. Regenerating backend.')
args = [config.substs['PYTHON'],
mozpath.join(self.topobjdir, 'config.status')]
self.run_process(args, cwd=self.topobjdir, pass_thru=True)
if (not config_rc and
self.backend_out_of_date(mozpath.join(self.topobjdir,
'backend.%sBackend' %
active_backend))):
print('Build configuration changed. Regenerating backend.')
args = [config.substs['PYTHON'],
mozpath.join(self.topobjdir, 'config.status')]
self.run_process(args, cwd=self.topobjdir, pass_thru=True)
if 'Make' not in active_backend:
# client.mk has its own handling of MOZ_PARALLEL_BUILD so the
# make backend can determine when to run in single-threaded mode
# or parallel mode. For other backends, we can pass in the value
@@ -1181,15 +1138,6 @@ class BuildDriver(MozbuildObject):
'instead of {target_pairs}.')
target_pairs = new_pairs
# Ensure build backend is up to date. The alternative is to
# have rules in the invoked Makefile to rebuild the build
# backend. But that involves make reinvoking itself and there
# are undesired side-effects of this. See bug 877308 for a
# comprehensive history lesson.
self._run_make(directory=self.topobjdir, target='backend',
line_handler=output.on_line, log=False,
print_directory=False, keep_going=keep_going)
# Build target pairs.
for make_dir, make_target in target_pairs:
# We don't display build status messages during partial

39
python/mozbuild/mozbuild/gen_test_backend.py Normal file
View File

@@ -0,0 +1,39 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import sys
from mozbuild.backend.test_manifest import TestManifestBackend
from mozbuild.base import BuildEnvironmentNotFoundException, MozbuildObject
from mozbuild.frontend.emitter import TreeMetadataEmitter
from mozbuild.frontend.reader import BuildReader, EmptyConfig
def gen_test_backend():
build_obj = MozbuildObject.from_environment()
try:
config = build_obj.config_environment
except BuildEnvironmentNotFoundException:
print("No build detected, test metadata may be incomplete.")
# If 'JS_STANDALONE' is set, tests that don't require an objdir won't
# be picked up due to bug 1345209.
substs = EmptyConfig.default_substs
if 'JS_STANDALONE' in substs:
del substs['JS_STANDALONE']
config = EmptyConfig(build_obj.topsrcdir, substs)
config.topobjdir = build_obj.topobjdir
reader = BuildReader(config)
emitter = TreeMetadataEmitter(config)
backend = TestManifestBackend(config)
context = reader.read_topsrcdir()
data = emitter.emit(context, emitfn=emitter._process_test_manifests)
backend.consume(data)
if __name__ == '__main__':
sys.exit(gen_test_backend())