Reapply "Bug 1967834 - moved process_gn_config call to generate_gn_config. r=ng"

This reverts commit 360d3a3311.

build/gn_processor.py

@@ -9,6 +9,7 @@ import subprocess
 import sys
 import tempfile
 from collections import defaultdict, deque
+from concurrent.futures import ProcessPoolExecutor, as_completed
 from copy import deepcopy
 from pathlib import Path
 from shutil import which
@@ -514,43 +515,11 @@ def find_common_attrs(config_attributes):
     return common_attrs
 
 
-def write_mozbuild(
+def write_mozbuild(topsrcdir, write_mozbuild_variables, relsrcdir, configs):
-    topsrcdir,
-    srcdir,
-    non_unified_sources,
-    gn_configs,
-    mozilla_flags,
-    write_mozbuild_variables,
-):
-    all_mozbuild_results = []
-
-    for gn_config in gn_configs:
-        mozbuild_attrs = process_gn_config(
-            gn_config,
-            topsrcdir,
-            srcdir,
-            non_unified_sources,
-            gn_config["sandbox_vars"],
-            mozilla_flags,
-        )
-        all_mozbuild_results.append(mozbuild_attrs)
-
-    # Translate {config -> {dirs -> build info}} into
-    #           {dirs -> [(config, build_info)]}
-    configs_by_dir = defaultdict(list)
-    for config_attrs in all_mozbuild_results:
-        mozbuild_args = config_attrs["mozbuild_args"]
-        dirs = config_attrs["dirs"]
-        for d, build_data in dirs.items():
-            configs_by_dir[d].append((mozbuild_args, build_data))
-
-    mozbuilds = set()
-    for relsrcdir, configs in sorted(configs_by_dir.items()):
     target_srcdir = mozpath.join(topsrcdir, relsrcdir)
     mkdir(target_srcdir)
 
     target_mozbuild = mozpath.join(target_srcdir, "moz.build")
-        mozbuilds.add(target_mozbuild)
     with open(target_mozbuild, "w") as fh:
         mb = MozbuildWriter(fh)
         mb.write(license_header)
@@ -571,16 +540,11 @@ def write_mozbuild(
         except KeyError:
             pass
         try:
-                if (
+            if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_LIBDRM_HANDLING"]:
-                    relsrcdir
-                    in write_mozbuild_variables["INCLUDE_SYSTEM_LIBDRM_HANDLING"]
-                ):
                 mb.write('CXXFLAGS += CONFIG["MOZ_LIBDRM_CFLAGS"]\n')
                 mb.write('if not CONFIG["MOZ_SYSTEM_LIBDRM"]:\n')
                 mb.write('    LOCAL_INCLUDES += [ "/third_party/drm/drm/",\n')
-                    mb.write(
+                mb.write('                        "/third_party/drm/drm/include/",\n')
-                        '                        "/third_party/drm/drm/include/",\n'
-                    )
                 mb.write(
                     '                        "/third_party/drm/drm/include/libdrm" ]\n'
                 )
@@ -597,20 +561,14 @@ def write_mozbuild(
         except KeyError:
             pass
         try:
-                if (
+            if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_LIBVPX_HANDLING"]:
-                    relsrcdir
-                    in write_mozbuild_variables["INCLUDE_SYSTEM_LIBVPX_HANDLING"]
-                ):
                 mb.write('if not CONFIG["MOZ_SYSTEM_LIBVPX"]:\n')
                 mb.write('    LOCAL_INCLUDES += [ "/media/libvpx/libvpx/" ]\n')
                 mb.write('    CXXFLAGS += CONFIG["MOZ_LIBVPX_CFLAGS"]\n')
         except KeyError:
             pass
         try:
-                if (
+            if relsrcdir in write_mozbuild_variables["INCLUDE_SYSTEM_DAV1D_HANDLING"]:
-                    relsrcdir
-                    in write_mozbuild_variables["INCLUDE_SYSTEM_DAV1D_HANDLING"]
-                ):
                 mb.write('if CONFIG["MOZ_SYSTEM_AV1"]:\n')
                 mb.write('    CXXFLAGS += CONFIG["MOZ_SYSTEM_DAV1D_CFLAGS"]\n')
                 mb.write('    CXXFLAGS += CONFIG["MOZ_SYSTEM_LIBAOM_CFLAGS"]\n')
@@ -656,6 +614,30 @@ def write_mozbuild(
                         mb.terminate_condition()
 
         mb.finalize()
+    return target_mozbuild
+
+
+def write_mozbuild_files(
+    topsrcdir,
+    srcdir,
+    all_mozbuild_results,
+    write_mozbuild_variables,
+):
+    # Translate {config -> {dirs -> build info}} into
+    #           {dirs -> [(config, build_info)]}
+    configs_by_dir = defaultdict(list)
+    for config_attrs in all_mozbuild_results:
+        mozbuild_args = config_attrs["mozbuild_args"]
+        dirs = config_attrs["dirs"]
+        for d, build_data in dirs.items():
+            configs_by_dir[d].append((mozbuild_args, build_data))
+
+    mozbuilds = set()
+    # threading this section did not produce noticeable speed gains
+    for relsrcdir, configs in sorted(configs_by_dir.items()):
+        mozbuilds.add(
+            write_mozbuild(topsrcdir, write_mozbuild_variables, relsrcdir, configs)
+        )
 
     # write the project moz.build file
     dirs_mozbuild = mozpath.join(srcdir, "moz.build")
@@ -713,6 +695,7 @@ def write_mozbuild(
 
 
 def generate_gn_config(
+    topsrcdir,
     build_root_dir,
     target_dir,
     gn_binary,
@@ -720,12 +703,17 @@ def generate_gn_config(
     sandbox_variables,
     gn_target,
     moz_build_flag,
+    non_unified_sources,
+    mozilla_flags,
 ):
     def str_for_arg(v):
         if v in (True, False):
             return str(v).lower()
         return '"%s"' % v
 
+    build_root_dir = topsrcdir / build_root_dir
+    srcdir = build_root_dir / target_dir
+
     input_variables = input_variables.copy()
     input_variables.update(
         {
@@ -785,6 +773,14 @@ def generate_gn_config(
                 input_variables,
                 gn_target,
             )
+            gn_config = process_gn_config(
+                gn_config,
+                topsrcdir,
+                srcdir,
+                non_unified_sources,
+                gn_config["sandbox_vars"],
+                mozilla_flags,
+            )
             return gn_config
 
 
@@ -833,26 +829,40 @@ def main():
                     vars_set.append(vars)
 
     gn_configs = []
-    for vars in vars_set:
+    NUM_WORKERS = 5
-        gn_configs.append(
+    with ProcessPoolExecutor(max_workers=NUM_WORKERS) as executor:
-            generate_gn_config(
+        # Submit tasks to the executor
-                topsrcdir / config["build_root_dir"],
+        futures = {
+            executor.submit(
+                generate_gn_config,
+                topsrcdir,
+                config["build_root_dir"],
                 config["target_dir"],
                 gn_binary,
                 vars,
                 config["gn_sandbox_variables"],
                 config["gn_target"],
                 config["moz_build_flag"],
-            )
+                config["non_unified_sources"],
-        )
+                config["mozilla_flags"],
+            ): vars
+            for vars in vars_set
+        }
+
+        # Process completed tasks as they finish
+        for future in as_completed(futures):
+            try:
+                gn_configs.append(future.result())
+            except Exception as e:
+                print(f"[Task] Task failed with exception: {e}")
+
+        print("All generation tasks have been processed.")
 
     print("Writing moz.build files")
-    write_mozbuild(
+    write_mozbuild_files(
         topsrcdir,
         topsrcdir / config["build_root_dir"] / config["target_dir"],
-        config["non_unified_sources"],
         gn_configs,
-        config["mozilla_flags"],
         config["write_mozbuild_variables"],
     )