From 29799d519a52a01a89f1d7bbc142c802e93504d4 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 7 Oct 2020 21:13:19 +0000 Subject: [PATCH] Bug 1669633 - Don't recurse into js/src for the python-part of configure. r=firefox-build-system-reviewers,rstewart Instead, we now run js/src/old-configure from the top-level configure after having run old-configure and extracted a few variables to inherit from it. Because we're now running from the top-level, $_objdir is always the top-level objdir, which simplifies some things. The topobjdir in js/src/config.status, however, needs to stay in js/src because of the build frontend expecting it there. When running js/src/old-configure, we used to need some special treatment for a large number of variables for historic reasons, where we'd take values from the assigned values before running old-configure for some, or from AC_SUBSTs after running old-configure. Now that both old-configure and js/src/old-configure get the same assignments from old-configure.vars, we don't need anything special for the former. And only a few remaining variables still need manual work for the latter. One notable difference, though, is that the new code doesn't try to avoid running js subconfigure, which added complexity, and was actually error-prone. Differential Revision: https://phabricator.services.mozilla.com/D92725 --- build/autoconf/altoptions.m4 | 2 +- build/autoconf/clang-plugin.m4 | 8 +- build/moz.configure/old.configure | 15 +++- configure.py | 33 +++++++- js/moz.configure | 9 +-- js/src/old-configure.in | 6 +- js/sub.configure | 124 +++++------------------------- 7 files changed, 67 insertions(+), 130 deletions(-) diff --git a/build/autoconf/altoptions.m4 b/build/autoconf/altoptions.m4 index b6e365bdcb11..ac016f4b911f 100644 --- a/build/autoconf/altoptions.m4 +++ b/build/autoconf/altoptions.m4 @@ -73,5 +73,5 @@ define(MOZ_ARG_HEADER, [# $1]) dnl MOZ_READ_MYCONFIG() - Read in 'myconfig.sh' file AC_DEFUN([MOZ_READ_MOZCONFIG], [AC_REQUIRE([AC_INIT_BINSH])dnl -. ./old-configure.vars +. $OLD_CONFIGURE_VARS ]) diff --git a/build/autoconf/clang-plugin.m4 b/build/autoconf/clang-plugin.m4 index 2000fe8ca22f..2bbb471e5b69 100644 --- a/build/autoconf/clang-plugin.m4 +++ b/build/autoconf/clang-plugin.m4 @@ -85,16 +85,12 @@ if test -n "$ENABLE_MOZSEARCH_PLUGIN"; then AC_MSG_ERROR([Can't use mozsearch plugin without --enable-clang-plugin.]) fi - dnl We use this construct rather than $_objdir to avoid getting /js/src in the - dnl path when compiling JS code. - OBJDIR="$(dirname $(dirname $(dirname $CLANG_PLUGIN)))" - CLANG_PLUGIN_FLAGS="$CLANG_PLUGIN_FLAGS -Xclang -add-plugin -Xclang mozsearch-index" dnl Parameters are: srcdir, outdir (path where output JSON is stored), objdir. CLANG_PLUGIN_FLAGS="$CLANG_PLUGIN_FLAGS -Xclang -plugin-arg-mozsearch-index -Xclang $_topsrcdir" - CLANG_PLUGIN_FLAGS="$CLANG_PLUGIN_FLAGS -Xclang -plugin-arg-mozsearch-index -Xclang $OBJDIR/mozsearch_index" - CLANG_PLUGIN_FLAGS="$CLANG_PLUGIN_FLAGS -Xclang -plugin-arg-mozsearch-index -Xclang $OBJDIR" + CLANG_PLUGIN_FLAGS="$CLANG_PLUGIN_FLAGS -Xclang -plugin-arg-mozsearch-index -Xclang $_objdir/mozsearch_index" + CLANG_PLUGIN_FLAGS="$CLANG_PLUGIN_FLAGS -Xclang -plugin-arg-mozsearch-index -Xclang $_objdir" AC_DEFINE(MOZ_MOZSEARCH_PLUGIN) fi diff --git a/build/moz.configure/old.configure b/build/moz.configure/old.configure index b58a8ede2e94..8c807c0794fa 100644 --- a/build/moz.configure/old.configure +++ b/build/moz.configure/old.configure @@ -127,8 +127,11 @@ def prepare_configure_options(host, target, all_options, *options): @template -def old_configure_for(old_configure_path): - @depends(prepare_configure, prepare_configure_options, altered_path, +def old_configure_for(old_configure_path, extra_env=None): + if extra_env is None: + extra_env = dependable(None) + + @depends(prepare_configure, prepare_configure_options, altered_path, extra_env, check_build_environment, old_configure_path, 'MOZILLABUILD', awk, m4, shell) @imports(_from='__builtin__', _import='compile') @imports(_from='__builtin__', _import='open') @@ -146,8 +149,8 @@ def old_configure_for(old_configure_path): @imports(_from='six', _import='exec_') @imports(_from='six', _import='iteritems') @imports(_from='six', _import='string_types') - def old_configure(prepare_configure, prepare_configure_options, altered_path, build_env, - old_configure, mozillabuild, awk, m4, shell): + def old_configure(prepare_configure, prepare_configure_options, altered_path, extra_env, + build_env, old_configure, mozillabuild, awk, m4, shell): # Use prepare_configure to make lint happy prepare_configure refresh = True @@ -235,6 +238,10 @@ def old_configure_for(old_configure_path): if altered_path: env['PATH'] = altered_path + if extra_env: + env.update(extra_env) + + env['OLD_CONFIGURE_VARS'] = os.path.join(build_env.topobjdir, 'old-configure.vars') proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) while True: diff --git a/configure.py b/configure.py index 8b7d583ff659..5b9c9494e58f 100644 --- a/configure.py +++ b/configure.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, print_function, unicode_literals import codecs +import errno import io import itertools import logging @@ -55,6 +56,31 @@ def main(argv): if sandbox._help: return 0 + logging.getLogger('moz.configure').info('Creating config.status') + + old_js_configure_substs = config.pop('OLD_JS_CONFIGURE_SUBSTS', None) + old_js_configure_defines = config.pop('OLD_JS_CONFIGURE_DEFINES', None) + if old_js_configure_substs or old_js_configure_defines: + js_config = config.copy() + pwd = os.getcwd() + try: + try: + os.makedirs('js/src') + except OSError as e: + if e.errno != errno.EEXIST: + raise + + os.chdir('js/src') + js_config['OLD_CONFIGURE_SUBSTS'] = old_js_configure_substs + js_config['OLD_CONFIGURE_DEFINES'] = old_js_configure_defines + # The build system frontend expects $objdir/js/src/config.status + # to have $objdir/js/src as topobjdir. + # We want forward slashes on all platforms. + js_config['TOPOBJDIR'] += '/js/src' + config_status(js_config, execute=False) + finally: + os.chdir(pwd) + return config_status(config) @@ -79,7 +105,7 @@ def check_unicode(obj): return True -def config_status(config): +def config_status(config, execute=True): # Sanitize config data to feed config.status # Ideally, all the backend and frontend code would handle the booleans, but # there are so many things involved, that it's easier to keep config.status @@ -139,7 +165,6 @@ def config_status(config): # Create config.status. Eventually, we'll want to just do the work it does # here, when we're able to skip configure tests/use cached results/not rely # on autoconf. - logging.getLogger('moz.configure').info('Creating config.status') with codecs.open('config.status', 'w', 'utf-8') as fh: fh.write(textwrap.dedent('''\ #!%(python)s @@ -152,7 +177,7 @@ def config_status(config): fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', " "'substs', 'mozconfig']") - if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'): + if execute: fh.write(textwrap.dedent(''' if __name__ == '__main__': from mozbuild.util import patch_main @@ -178,7 +203,7 @@ def config_status(config): # Other things than us are going to run this file, so we need to give it # executable permissions. os.chmod('config.status', 0o755) - if config.get('MOZ_BUILD_APP') != 'js' or config.get('JS_STANDALONE'): + if execute: from mozbuild.config_status import config_status return config_status(args=[], **sanitized_config) return 0 diff --git a/js/moz.configure b/js/moz.configure index a6882315b02d..928f5321117b 100644 --- a/js/moz.configure +++ b/js/moz.configure @@ -433,13 +433,8 @@ build_ctypes = depends_if('--enable-ctypes')(lambda _: True) set_config('BUILD_CTYPES', build_ctypes) set_define('BUILD_CTYPES', build_ctypes) -@depends(build_ctypes, building_js) -def js_has_ctypes(ctypes, js): - if ctypes and js: - return True - -set_config('JS_HAS_CTYPES', js_has_ctypes) -set_define('JS_HAS_CTYPES', js_has_ctypes) +set_config('JS_HAS_CTYPES', build_ctypes) +set_define('JS_HAS_CTYPES', build_ctypes) @depends('--enable-ctypes', '--enable-compile-environment') def ctypes_and_compile_environment(ctypes, compile_environment): diff --git a/js/src/old-configure.in b/js/src/old-configure.in index b4757720b287..bae689ad4d89 100644 --- a/js/src/old-configure.in +++ b/js/src/old-configure.in @@ -64,11 +64,7 @@ if test -z "$JS_STANDALONE"; then fi AC_SUBST(autoconfmk) -if test -n "$JS_STANDALONE"; then - jsconfdefs=$_objdir/js/src/js-confdefs.h -else - jsconfdefs=$_objdir/js-confdefs.h -fi +jsconfdefs=$_objdir/js/src/js-confdefs.h MOZ_ANDROID_NDK diff --git a/js/sub.configure b/js/sub.configure index 0b2d81f01eb8..a972f3da9f5e 100644 --- a/js/sub.configure +++ b/js/sub.configure @@ -4,23 +4,11 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -@depends(host_for_sub_configure, target_for_sub_configure, check_build_environment, - js_configure_args, prepare_mozconfig, old_configure, - old_configure_assignments, '--cache-file') -@imports('errno') +@depends(check_build_environment) @imports('logging') -@imports('os') -@imports('pickle') -@imports('sys') -@imports(_from='__main__', _import='config_status') -@imports(_from='__builtin__', _import='OSError') -@imports(_from='__builtin__', _import='open') @imports(_from='__builtin__', _import='object') -@imports(_from='mozbuild.configure', _import='ConfigureSandbox') @imports(_from='mozbuild.configure.util', _import='ConfigureOutputHandler') -def js_subconfigure(host, target, build_env, js_configure_args, mozconfig, - old_configure, old_configure_assignments, cache_file): - +def old_js_configure(build_env): class PrefixOutput(object): def __init__(self, prefix, fh): self._fh = fh @@ -44,43 +32,28 @@ def js_subconfigure(host, target, build_env, js_configure_args, mozconfig, handler.setFormatter(formatter) if isinstance(handler, ConfigureOutputHandler): handler._stdout = PrefixOutput('js/src> ', handler._stdout) + return os.path.join(build_env.topsrcdir, 'js', 'src', 'old-configure') - substs = dict(old_configure.substs) - assignments = dict(old_configure_assignments) - environ = dict(os.environ) - - options = [host, target] + js_configure_args - - environ['MOZILLA_CENTRAL_PATH'] = build_env.topsrcdir - if 'MOZ_BUILD_APP' in environ: - del environ['MOZ_BUILD_APP'] +@depends(old_configure.substs, mozconfig) +def old_js_configure_env(substs, mozconfig): + substs = dict(substs) # Here, we mimic what we used to do from old-configure, which makes this # all awkward. - # The following variables were saved at the beginning of old-configure, - # and restored before invoking the subconfigure. Which means their value - # should be taken from the old_configure_assignments or mozconfig. - from_assignment = set( - ('CC', 'CXX', 'CPPFLAGS', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'HOST_CC', - 'HOST_CXXFLAGS', 'HOST_LDFLAGS')) - # Variables that were explicitly exported from old-configure, and those # explicitly set in the environment when invoking old-configure, were # automatically inherited from subconfigure. We assume the relevant ones # have a corresponding AC_SUBST in old-configure, making them available # in `substs`. + extra_env = {} + for var in ( - 'MOZ_SYSTEM_ZLIB', 'MOZ_ZLIB_CFLAGS', 'MOZ_ZLIB_LIBS', - 'MOZ_DEV_EDITION', 'STLPORT_LIBS', 'DIST', 'MOZ_LINKER', - 'ZLIB_IN_MOZGLUE', 'RANLIB', 'AR', 'CPP', 'CC', 'CXX', 'CPPFLAGS', - 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'HOST_CC', 'HOST_CXX', 'HOST_CPPFLAGS', - 'HOST_CXXFLAGS', 'HOST_LDFLAGS' + 'MOZ_DEV_EDITION', 'STLPORT_LIBS', 'MOZ_LINKER', 'ZLIB_IN_MOZGLUE', + 'RANLIB', ): - if var not in from_assignment and var in substs: + if var in substs: value = substs[var] - elif var in assignments: - value = assignments[var] elif mozconfig and var in mozconfig and \ not mozconfig[var][1].startswith('removed'): value = mozconfig[var][0] @@ -88,80 +61,25 @@ def js_subconfigure(host, target, build_env, js_configure_args, mozconfig, continue if isinstance(value, list): value = ' '.join(value) - environ[var] = value + extra_env[var] = value - options.append('JS_STANDALONE=') + return extra_env - srcdir = os.path.join(build_env.topsrcdir, 'js', 'src') - objdir = os.path.join(build_env.topobjdir, 'js', 'src') - data_file = os.path.join(objdir, 'configure.pkl') - previous_args = None - if os.path.exists(data_file): - with open(data_file, 'rb') as f: - previous_args = pickle.load(f) +old_js_configure = old_configure_for(old_js_configure, extra_env=old_js_configure_env) +set_config('OLD_JS_CONFIGURE_SUBSTS', old_js_configure.substs) +set_config('OLD_JS_CONFIGURE_DEFINES', old_js_configure.defines) - cache_file = cache_file[0] if cache_file else './config.cache' - cache_file = os.path.join(build_env.topobjdir, cache_file) - - try: - os.makedirs(objdir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - with open(data_file, 'wb') as f: - pickle.dump(options, f) - - # Only run configure if one of the following is true: - # - config.status doesn't exist - # - config.status is older than an input to configure - # - the configure arguments changed - configure = os.path.join(srcdir, 'old-configure') - config_status_path = os.path.join(objdir, 'config.status') - skip_configure = True - if not os.path.exists(config_status_path): - skip_configure = False - else: - config_status_deps = os.path.join(objdir, 'config_status_deps.in') - if not os.path.exists(config_status_deps): - skip_configure = False - else: - with open(config_status_deps, 'r') as fh: - dep_files = fh.read().splitlines() + [configure] - if (any(not os.path.exists(f) or - (os.path.getmtime(config_status_path) < os.path.getmtime(f)) - for f in dep_files) or - ((previous_args or options) != options)): - skip_configure = False - - ret = 0 - if not skip_configure: - oldpwd = os.getcwd() - os.chdir(objdir) - command = [ - os.path.join(build_env.topsrcdir, 'configure.py'), - '--enable-project=js', - ] - environ['OLD_CONFIGURE'] = os.path.join( - os.path.dirname(configure), 'old-configure') - command += options - command += ['--cache-file=%s' % cache_file] - - log.info('configuring') - log.info('running %s' % ' '.join(command[:-1])) - config = {} - sandbox = ConfigureSandbox(config, environ, command, logger=logger) - sandbox.run(os.path.join(build_env.topsrcdir, 'moz.configure')) - ret = config_status(config) - os.chdir(oldpwd) +@dependable +@imports('logging') +@imports(_from='mozbuild.configure.util', _import='ConfigureOutputHandler') +def post_old_js_configure(): # Restore unprefixed logging. formatter = logging.Formatter('%(levelname)s: %(message)s') + logger = logging.getLogger('moz.configure') for handler in logger.handlers: handler.setFormatter(formatter) if isinstance(handler, ConfigureOutputHandler): handler._stdout.flush() handler._stdout = handler._stdout._fh - - return ret