diff --git a/build/gn_processor.py b/build/gn_processor.py index 5d31a43e44bd..7da0a7b5c44f 100644 --- a/build/gn_processor.py +++ b/build/gn_processor.py @@ -9,6 +9,7 @@ import subprocess import sys import tempfile from collections import defaultdict, deque +from concurrent.futures import ProcessPoolExecutor, as_completed from copy import deepcopy from pathlib import Path from shutil import which @@ -514,27 +515,114 @@ def find_common_attrs(config_attributes): return common_attrs -def write_mozbuild( +def write_mozbuild(topsrcdir, write_mozbuild_variables, relsrcdir, configs): + target_srcdir = mozpath.join(topsrcdir, relsrcdir) + mkdir(target_srcdir) + + target_mozbuild = mozpath.join(target_srcdir, "moz.build") + with open(target_mozbuild, "w") as fh: + mb = MozbuildWriter(fh) + mb.write(license_header) + mb.write("\n") + mb.write(generated_header) + + try: + if relsrcdir in write_mozbuild_variables["INCLUDE_TK_CFLAGS_DIRS"]: + mb.write('if CONFIG["MOZ_WIDGET_TOOLKIT"] == "gtk":\n') + mb.write(' CXXFLAGS += CONFIG["MOZ_GTK3_CFLAGS"]\n') + except KeyError: + pass + try: + if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_GBM_HANDLING"]: + mb.write('CXXFLAGS += CONFIG["MOZ_GBM_CFLAGS"]\n') + mb.write('if not CONFIG["MOZ_SYSTEM_GBM"]:\n') + mb.write(' LOCAL_INCLUDES += [ "/third_party/gbm/gbm/" ]\n') + except KeyError: + pass + try: + if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_LIBDRM_HANDLING"]: + mb.write('CXXFLAGS += CONFIG["MOZ_LIBDRM_CFLAGS"]\n') + mb.write('if not CONFIG["MOZ_SYSTEM_LIBDRM"]:\n') + mb.write(' LOCAL_INCLUDES += [ "/third_party/drm/drm/",\n') + mb.write(' "/third_party/drm/drm/include/",\n') + mb.write( + ' "/third_party/drm/drm/include/libdrm" ]\n' + ) + except KeyError: + pass + try: + if ( + relsrcdir + in write_mozbuild_variables["INCLUDE_SYSTEM_PIPEWIRE_HANDLING"] + ): + mb.write('CXXFLAGS += CONFIG["MOZ_PIPEWIRE_CFLAGS"]\n') + mb.write('if not CONFIG["MOZ_SYSTEM_PIPEWIRE"]:\n') + mb.write(' LOCAL_INCLUDES += [ "/third_party/pipewire/" ]\n') + except KeyError: + pass + try: + if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_LIBVPX_HANDLING"]: + mb.write('if not CONFIG["MOZ_SYSTEM_LIBVPX"]:\n') + mb.write(' LOCAL_INCLUDES += [ "/media/libvpx/libvpx/" ]\n') + mb.write(' CXXFLAGS += CONFIG["MOZ_LIBVPX_CFLAGS"]\n') + except KeyError: + pass + try: + if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_DAV1D_HANDLING"]: + mb.write('if CONFIG["MOZ_SYSTEM_AV1"]:\n') + mb.write(' CXXFLAGS += CONFIG["MOZ_SYSTEM_DAV1D_CFLAGS"]\n') + mb.write(' CXXFLAGS += CONFIG["MOZ_SYSTEM_LIBAOM_CFLAGS"]\n') + except KeyError: + pass + + all_args = [args for args, _ in configs] + + # Start with attributes that will be a part of the mozconfig + # for every configuration, then factor by other potentially useful + # combinations. + # FIXME: this is a time-bomb. See bug 1775202. + for attrs in ( + (), + ("MOZ_DEBUG",), + ("OS_TARGET",), + ("TARGET_CPU",), + ("MOZ_DEBUG", "OS_TARGET"), + ("OS_TARGET", "MOZ_X11"), + ("OS_TARGET", "TARGET_CPU"), + ("OS_TARGET", "TARGET_CPU", "MOZ_X11"), + ("OS_TARGET", "TARGET_CPU", "MOZ_DEBUG"), + ("OS_TARGET", "TARGET_CPU", "MOZ_DEBUG", "MOZ_X11"), + ): + conditions = set() + for args in all_args: + cond = tuple((k, args.get(k) or "") for k in attrs) + conditions.add(cond) + + for cond in sorted(conditions): + common_attrs = find_common_attrs( + [ + attrs + for args, attrs in configs + if all((args.get(k) or "") == v for k, v in cond) + ] + ) + if any(common_attrs.values()): + if cond: + mb.write_condition(dict(cond)) + mb.write_attrs(common_attrs) + if cond: + mb.terminate_condition() + + mb.finalize() + return target_mozbuild + + +def write_mozbuild_files( topsrcdir, srcdir, - non_unified_sources, - gn_configs, - mozilla_flags, + all_mozbuild_results, write_mozbuild_variables, ): - all_mozbuild_results = [] - - for gn_config in gn_configs: - mozbuild_attrs = process_gn_config( - gn_config, - topsrcdir, - srcdir, - non_unified_sources, - gn_config["sandbox_vars"], - mozilla_flags, - ) - all_mozbuild_results.append(mozbuild_attrs) - # Translate {config -> {dirs -> build info}} into # {dirs -> [(config, build_info)]} configs_by_dir = defaultdict(list) @@ -545,117 +633,11 @@ def write_mozbuild( configs_by_dir[d].append((mozbuild_args, build_data)) mozbuilds = set() + # threading this section did not produce noticeable speed gains for relsrcdir, configs in sorted(configs_by_dir.items()): - target_srcdir = mozpath.join(topsrcdir, relsrcdir) - mkdir(target_srcdir) - - target_mozbuild = mozpath.join(target_srcdir, "moz.build") - mozbuilds.add(target_mozbuild) - with open(target_mozbuild, "w") as fh: - mb = MozbuildWriter(fh) - mb.write(license_header) - mb.write("\n") - mb.write(generated_header) - - try: - if relsrcdir in write_mozbuild_variables["INCLUDE_TK_CFLAGS_DIRS"]: - mb.write('if CONFIG["MOZ_WIDGET_TOOLKIT"] == "gtk":\n') - mb.write(' CXXFLAGS += CONFIG["MOZ_GTK3_CFLAGS"]\n') - except KeyError: - pass - try: - if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_GBM_HANDLING"]: - mb.write('CXXFLAGS += CONFIG["MOZ_GBM_CFLAGS"]\n') - mb.write('if not CONFIG["MOZ_SYSTEM_GBM"]:\n') - mb.write(' LOCAL_INCLUDES += [ "/third_party/gbm/gbm/" ]\n') - except KeyError: - pass - try: - if ( - relsrcdir - in write_mozbuild_variables["INCLUDE_SYSTEM_LIBDRM_HANDLING"] - ): - mb.write('CXXFLAGS += CONFIG["MOZ_LIBDRM_CFLAGS"]\n') - mb.write('if not CONFIG["MOZ_SYSTEM_LIBDRM"]:\n') - mb.write(' LOCAL_INCLUDES += [ "/third_party/drm/drm/",\n') - mb.write( - ' "/third_party/drm/drm/include/",\n' - ) - mb.write( - ' "/third_party/drm/drm/include/libdrm" ]\n' - ) - except KeyError: - pass - try: - if ( - relsrcdir - in write_mozbuild_variables["INCLUDE_SYSTEM_PIPEWIRE_HANDLING"] - ): - mb.write('CXXFLAGS += CONFIG["MOZ_PIPEWIRE_CFLAGS"]\n') - mb.write('if not CONFIG["MOZ_SYSTEM_PIPEWIRE"]:\n') - mb.write(' LOCAL_INCLUDES += [ "/third_party/pipewire/" ]\n') - except KeyError: - pass - try: - if ( - relsrcdir - in write_mozbuild_variables["INCLUDE_SYSTEM_LIBVPX_HANDLING"] - ): - mb.write('if not CONFIG["MOZ_SYSTEM_LIBVPX"]:\n') - mb.write(' LOCAL_INCLUDES += [ "/media/libvpx/libvpx/" ]\n') - mb.write(' CXXFLAGS += CONFIG["MOZ_LIBVPX_CFLAGS"]\n') - except KeyError: - pass - try: - if ( - relsrcdir - in write_mozbuild_variables["INCLUDE_SYSTEM_DAV1D_HANDLING"] - ): - mb.write('if CONFIG["MOZ_SYSTEM_AV1"]:\n') - mb.write(' CXXFLAGS += CONFIG["MOZ_SYSTEM_DAV1D_CFLAGS"]\n') - mb.write(' CXXFLAGS += CONFIG["MOZ_SYSTEM_LIBAOM_CFLAGS"]\n') - except KeyError: - pass - - all_args = [args for args, _ in configs] - - # Start with attributes that will be a part of the mozconfig - # for every configuration, then factor by other potentially useful - # combinations. - # FIXME: this is a time-bomb. See bug 1775202. - for attrs in ( - (), - ("MOZ_DEBUG",), - ("OS_TARGET",), - ("TARGET_CPU",), - ("MOZ_DEBUG", "OS_TARGET"), - ("OS_TARGET", "MOZ_X11"), - ("OS_TARGET", "TARGET_CPU"), - ("OS_TARGET", "TARGET_CPU", "MOZ_X11"), - ("OS_TARGET", "TARGET_CPU", "MOZ_DEBUG"), - ("OS_TARGET", "TARGET_CPU", "MOZ_DEBUG", "MOZ_X11"), - ): - conditions = set() - for args in all_args: - cond = tuple((k, args.get(k) or "") for k in attrs) - conditions.add(cond) - - for cond in sorted(conditions): - common_attrs = find_common_attrs( - [ - attrs - for args, attrs in configs - if all((args.get(k) or "") == v for k, v in cond) - ] - ) - if any(common_attrs.values()): - if cond: - mb.write_condition(dict(cond)) - mb.write_attrs(common_attrs) - if cond: - mb.terminate_condition() - - mb.finalize() + mozbuilds.add( + write_mozbuild(topsrcdir, write_mozbuild_variables, relsrcdir, configs) + ) # write the project moz.build file dirs_mozbuild = mozpath.join(srcdir, "moz.build") @@ -713,6 +695,7 @@ def write_mozbuild( def generate_gn_config( + topsrcdir, build_root_dir, target_dir, gn_binary, @@ -720,12 +703,17 @@ def generate_gn_config( sandbox_variables, gn_target, moz_build_flag, + non_unified_sources, + mozilla_flags, ): def str_for_arg(v): if v in (True, False): return str(v).lower() return '"%s"' % v + build_root_dir = topsrcdir / build_root_dir + srcdir = build_root_dir / target_dir + input_variables = input_variables.copy() input_variables.update( { @@ -785,6 +773,14 @@ def generate_gn_config( input_variables, gn_target, ) + gn_config = process_gn_config( + gn_config, + topsrcdir, + srcdir, + non_unified_sources, + gn_config["sandbox_vars"], + mozilla_flags, + ) return gn_config @@ -833,26 +829,40 @@ def main(): vars_set.append(vars) gn_configs = [] - for vars in vars_set: - gn_configs.append( - generate_gn_config( - topsrcdir / config["build_root_dir"], + NUM_WORKERS = 5 + with ProcessPoolExecutor(max_workers=NUM_WORKERS) as executor: + # Submit tasks to the executor + futures = { + executor.submit( + generate_gn_config, + topsrcdir, + config["build_root_dir"], config["target_dir"], gn_binary, vars, config["gn_sandbox_variables"], config["gn_target"], config["moz_build_flag"], - ) - ) + config["non_unified_sources"], + config["mozilla_flags"], + ): vars + for vars in vars_set + } + + # Process completed tasks as they finish + for future in as_completed(futures): + try: + gn_configs.append(future.result()) + except Exception as e: + print(f"[Task] Task failed with exception: {e}") + + print("All generation tasks have been processed.") print("Writing moz.build files") - write_mozbuild( + write_mozbuild_files( topsrcdir, topsrcdir / config["build_root_dir"] / config["target_dir"], - config["non_unified_sources"], gn_configs, - config["mozilla_flags"], config["write_mozbuild_variables"], )