Reapply "Bug 1967834 - moved process_gn_config call to generate_gn_config. r=ng"

This reverts commit 360d3a3311.

build/gn_processor.py

@@ -9,6 +9,7 @@ import subprocess
 import sys
 import tempfile
 from collections import defaultdict, deque
+from concurrent.futures import ProcessPoolExecutor, as_completed
 from copy import deepcopy
 from pathlib import Path
 from shutil import which
@@ -514,27 +515,114 @@ def find_common_attrs(config_attributes):
     return common_attrs
 
 
-def write_mozbuild(
+def write_mozbuild(topsrcdir, write_mozbuild_variables, relsrcdir, configs):
+    target_srcdir = mozpath.join(topsrcdir, relsrcdir)
+    mkdir(target_srcdir)
+
+    target_mozbuild = mozpath.join(target_srcdir, "moz.build")
+    with open(target_mozbuild, "w") as fh:
+        mb = MozbuildWriter(fh)
+        mb.write(license_header)
+        mb.write("\n")
+        mb.write(generated_header)
+
+        try:
+            if relsrcdir in write_mozbuild_variables["INCLUDE_TK_CFLAGS_DIRS"]:
+                mb.write('if CONFIG["MOZ_WIDGET_TOOLKIT"] == "gtk":\n')
+                mb.write('    CXXFLAGS += CONFIG["MOZ_GTK3_CFLAGS"]\n')
+        except KeyError:
+            pass
+        try:
+            if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_GBM_HANDLING"]:
+                mb.write('CXXFLAGS += CONFIG["MOZ_GBM_CFLAGS"]\n')
+                mb.write('if not CONFIG["MOZ_SYSTEM_GBM"]:\n')
+                mb.write('    LOCAL_INCLUDES += [ "/third_party/gbm/gbm/" ]\n')
+        except KeyError:
+            pass
+        try:
+            if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_LIBDRM_HANDLING"]:
+                mb.write('CXXFLAGS += CONFIG["MOZ_LIBDRM_CFLAGS"]\n')
+                mb.write('if not CONFIG["MOZ_SYSTEM_LIBDRM"]:\n')
+                mb.write('    LOCAL_INCLUDES += [ "/third_party/drm/drm/",\n')
+                mb.write('                        "/third_party/drm/drm/include/",\n')
+                mb.write(
+                    '                        "/third_party/drm/drm/include/libdrm" ]\n'
+                )
+        except KeyError:
+            pass
+        try:
+            if (
+                relsrcdir
+                in write_mozbuild_variables["INCLUDE_SYSTEM_PIPEWIRE_HANDLING"]
+            ):
+                mb.write('CXXFLAGS += CONFIG["MOZ_PIPEWIRE_CFLAGS"]\n')
+                mb.write('if not CONFIG["MOZ_SYSTEM_PIPEWIRE"]:\n')
+                mb.write('    LOCAL_INCLUDES += [ "/third_party/pipewire/" ]\n')
+        except KeyError:
+            pass
+        try:
+            if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_LIBVPX_HANDLING"]:
+                mb.write('if not CONFIG["MOZ_SYSTEM_LIBVPX"]:\n')
+                mb.write('    LOCAL_INCLUDES += [ "/media/libvpx/libvpx/" ]\n')
+                mb.write('    CXXFLAGS += CONFIG["MOZ_LIBVPX_CFLAGS"]\n')
+        except KeyError:
+            pass
+        try:
+            if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_DAV1D_HANDLING"]:
+                mb.write('if CONFIG["MOZ_SYSTEM_AV1"]:\n')
+                mb.write('    CXXFLAGS += CONFIG["MOZ_SYSTEM_DAV1D_CFLAGS"]\n')
+                mb.write('    CXXFLAGS += CONFIG["MOZ_SYSTEM_LIBAOM_CFLAGS"]\n')
+        except KeyError:
+            pass
+
+        all_args = [args for args, _ in configs]
+
+        # Start with attributes that will be a part of the mozconfig
+        # for every configuration, then factor by other potentially useful
+        # combinations.
+        # FIXME: this is a time-bomb. See bug 1775202.
+        for attrs in (
+            (),
+            ("MOZ_DEBUG",),
+            ("OS_TARGET",),
+            ("TARGET_CPU",),
+            ("MOZ_DEBUG", "OS_TARGET"),
+            ("OS_TARGET", "MOZ_X11"),
+            ("OS_TARGET", "TARGET_CPU"),
+            ("OS_TARGET", "TARGET_CPU", "MOZ_X11"),
+            ("OS_TARGET", "TARGET_CPU", "MOZ_DEBUG"),
+            ("OS_TARGET", "TARGET_CPU", "MOZ_DEBUG", "MOZ_X11"),
+        ):
+            conditions = set()
+            for args in all_args:
+                cond = tuple((k, args.get(k) or "") for k in attrs)
+                conditions.add(cond)
+
+            for cond in sorted(conditions):
+                common_attrs = find_common_attrs(
+                    [
+                        attrs
+                        for args, attrs in configs
+                        if all((args.get(k) or "") == v for k, v in cond)
+                    ]
+                )
+                if any(common_attrs.values()):
+                    if cond:
+                        mb.write_condition(dict(cond))
+                    mb.write_attrs(common_attrs)
+                    if cond:
+                        mb.terminate_condition()
+
+        mb.finalize()
+    return target_mozbuild
+
+
+def write_mozbuild_files(
     topsrcdir,
     srcdir,
-    non_unified_sources,
+    all_mozbuild_results,
-    gn_configs,
-    mozilla_flags,
     write_mozbuild_variables,
 ):
-    all_mozbuild_results = []
-
-    for gn_config in gn_configs:
-        mozbuild_attrs = process_gn_config(
-            gn_config,
-            topsrcdir,
-            srcdir,
-            non_unified_sources,
-            gn_config["sandbox_vars"],
-            mozilla_flags,
-        )
-        all_mozbuild_results.append(mozbuild_attrs)
-
     # Translate {config -> {dirs -> build info}} into
     #           {dirs -> [(config, build_info)]}
     configs_by_dir = defaultdict(list)
@@ -545,117 +633,11 @@ def write_mozbuild(
             configs_by_dir[d].append((mozbuild_args, build_data))
 
     mozbuilds = set()
+    # threading this section did not produce noticeable speed gains
     for relsrcdir, configs in sorted(configs_by_dir.items()):
-        target_srcdir = mozpath.join(topsrcdir, relsrcdir)
+        mozbuilds.add(
-        mkdir(target_srcdir)
+            write_mozbuild(topsrcdir, write_mozbuild_variables, relsrcdir, configs)
-
+        )
-        target_mozbuild = mozpath.join(target_srcdir, "moz.build")
-        mozbuilds.add(target_mozbuild)
-        with open(target_mozbuild, "w") as fh:
-            mb = MozbuildWriter(fh)
-            mb.write(license_header)
-            mb.write("\n")
-            mb.write(generated_header)
-
-            try:
-                if relsrcdir in write_mozbuild_variables["INCLUDE_TK_CFLAGS_DIRS"]:
-                    mb.write('if CONFIG["MOZ_WIDGET_TOOLKIT"] == "gtk":\n')
-                    mb.write('    CXXFLAGS += CONFIG["MOZ_GTK3_CFLAGS"]\n')
-            except KeyError:
-                pass
-            try:
-                if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_GBM_HANDLING"]:
-                    mb.write('CXXFLAGS += CONFIG["MOZ_GBM_CFLAGS"]\n')
-                    mb.write('if not CONFIG["MOZ_SYSTEM_GBM"]:\n')
-                    mb.write('    LOCAL_INCLUDES += [ "/third_party/gbm/gbm/" ]\n')
-            except KeyError:
-                pass
-            try:
-                if (
-                    relsrcdir
-                    in write_mozbuild_variables["INCLUDE_SYSTEM_LIBDRM_HANDLING"]
-                ):
-                    mb.write('CXXFLAGS += CONFIG["MOZ_LIBDRM_CFLAGS"]\n')
-                    mb.write('if not CONFIG["MOZ_SYSTEM_LIBDRM"]:\n')
-                    mb.write('    LOCAL_INCLUDES += [ "/third_party/drm/drm/",\n')
-                    mb.write(
-                        '                        "/third_party/drm/drm/include/",\n'
-                    )
-                    mb.write(
-                        '                        "/third_party/drm/drm/include/libdrm" ]\n'
-                    )
-            except KeyError:
-                pass
-            try:
-                if (
-                    relsrcdir
-                    in write_mozbuild_variables["INCLUDE_SYSTEM_PIPEWIRE_HANDLING"]
-                ):
-                    mb.write('CXXFLAGS += CONFIG["MOZ_PIPEWIRE_CFLAGS"]\n')
-                    mb.write('if not CONFIG["MOZ_SYSTEM_PIPEWIRE"]:\n')
-                    mb.write('    LOCAL_INCLUDES += [ "/third_party/pipewire/" ]\n')
-            except KeyError:
-                pass
-            try:
-                if (
-                    relsrcdir
-                    in write_mozbuild_variables["INCLUDE_SYSTEM_LIBVPX_HANDLING"]
-                ):
-                    mb.write('if not CONFIG["MOZ_SYSTEM_LIBVPX"]:\n')
-                    mb.write('    LOCAL_INCLUDES += [ "/media/libvpx/libvpx/" ]\n')
-                    mb.write('    CXXFLAGS += CONFIG["MOZ_LIBVPX_CFLAGS"]\n')
-            except KeyError:
-                pass
-            try:
-                if (
-                    relsrcdir
-                    in write_mozbuild_variables["INCLUDE_SYSTEM_DAV1D_HANDLING"]
-                ):
-                    mb.write('if CONFIG["MOZ_SYSTEM_AV1"]:\n')
-                    mb.write('    CXXFLAGS += CONFIG["MOZ_SYSTEM_DAV1D_CFLAGS"]\n')
-                    mb.write('    CXXFLAGS += CONFIG["MOZ_SYSTEM_LIBAOM_CFLAGS"]\n')
-            except KeyError:
-                pass
-
-            all_args = [args for args, _ in configs]
-
-            # Start with attributes that will be a part of the mozconfig
-            # for every configuration, then factor by other potentially useful
-            # combinations.
-            # FIXME: this is a time-bomb. See bug 1775202.
-            for attrs in (
-                (),
-                ("MOZ_DEBUG",),
-                ("OS_TARGET",),
-                ("TARGET_CPU",),
-                ("MOZ_DEBUG", "OS_TARGET"),
-                ("OS_TARGET", "MOZ_X11"),
-                ("OS_TARGET", "TARGET_CPU"),
-                ("OS_TARGET", "TARGET_CPU", "MOZ_X11"),
-                ("OS_TARGET", "TARGET_CPU", "MOZ_DEBUG"),
-                ("OS_TARGET", "TARGET_CPU", "MOZ_DEBUG", "MOZ_X11"),
-            ):
-                conditions = set()
-                for args in all_args:
-                    cond = tuple((k, args.get(k) or "") for k in attrs)
-                    conditions.add(cond)
-
-                for cond in sorted(conditions):
-                    common_attrs = find_common_attrs(
-                        [
-                            attrs
-                            for args, attrs in configs
-                            if all((args.get(k) or "") == v for k, v in cond)
-                        ]
-                    )
-                    if any(common_attrs.values()):
-                        if cond:
-                            mb.write_condition(dict(cond))
-                        mb.write_attrs(common_attrs)
-                        if cond:
-                            mb.terminate_condition()
-
-            mb.finalize()
 
     # write the project moz.build file
     dirs_mozbuild = mozpath.join(srcdir, "moz.build")
@@ -713,6 +695,7 @@ def write_mozbuild(
 
 
 def generate_gn_config(
+    topsrcdir,
     build_root_dir,
     target_dir,
     gn_binary,
@@ -720,12 +703,17 @@ def generate_gn_config(
     sandbox_variables,
     gn_target,
     moz_build_flag,
+    non_unified_sources,
+    mozilla_flags,
 ):
     def str_for_arg(v):
         if v in (True, False):
             return str(v).lower()
         return '"%s"' % v
 
+    build_root_dir = topsrcdir / build_root_dir
+    srcdir = build_root_dir / target_dir
+
     input_variables = input_variables.copy()
     input_variables.update(
         {
@@ -785,6 +773,14 @@ def generate_gn_config(
                 input_variables,
                 gn_target,
             )
+            gn_config = process_gn_config(
+                gn_config,
+                topsrcdir,
+                srcdir,
+                non_unified_sources,
+                gn_config["sandbox_vars"],
+                mozilla_flags,
+            )
             return gn_config
 
 
@@ -833,26 +829,40 @@ def main():
                     vars_set.append(vars)
 
     gn_configs = []
-    for vars in vars_set:
+    NUM_WORKERS = 5
-        gn_configs.append(
+    with ProcessPoolExecutor(max_workers=NUM_WORKERS) as executor:
-            generate_gn_config(
+        # Submit tasks to the executor
-                topsrcdir / config["build_root_dir"],
+        futures = {
+            executor.submit(
+                generate_gn_config,
+                topsrcdir,
+                config["build_root_dir"],
                 config["target_dir"],
                 gn_binary,
                 vars,
                 config["gn_sandbox_variables"],
                 config["gn_target"],
                 config["moz_build_flag"],
-            )
+                config["non_unified_sources"],
-        )
+                config["mozilla_flags"],
+            ): vars
+            for vars in vars_set
+        }
+
+        # Process completed tasks as they finish
+        for future in as_completed(futures):
+            try:
+                gn_configs.append(future.result())
+            except Exception as e:
+                print(f"[Task] Task failed with exception: {e}")
+
+        print("All generation tasks have been processed.")
 
     print("Writing moz.build files")
-    write_mozbuild(
+    write_mozbuild_files(
         topsrcdir,
         topsrcdir / config["build_root_dir"] / config["target_dir"],
-        config["non_unified_sources"],
         gn_configs,
-        config["mozilla_flags"],
         config["write_mozbuild_variables"],
     )