Bug 1726782 - Create a new task generating a clang profile for PGO. r=firefox-build-system-reviewers,mhentges

The new task builds a stage 2 compiler and uses it to generate the
profile as stage 3, and the profile is used to feed the now separate
stage 4 clang build. This will allow us to use this same profile to
build mac clangs with PGO.

Differential Revision: https://phabricator.services.mozilla.com/D138511

build/build-clang/build-clang.py

@@ -199,7 +199,7 @@ def build_one_stage(
     libcxx_include_dir,
     build_wasm,
     is_final_stage=False,
-    pgo_phase=None,
+    profile=None,
 ):
     if not os.path.exists(stage_dir):
         os.mkdir(stage_dir)
@@ -316,15 +316,15 @@ def build_one_stage(
                 "-DDARWIN_macosx_OVERRIDE_SDK_VERSION=%s"
                 % os.environ["MACOSX_DEPLOYMENT_TARGET"],
             ]
-        if pgo_phase == "gen":
+        if profile == "gen":
             # Per https://releases.llvm.org/10.0.0/docs/HowToBuildWithPGO.html
             cmake_args += [
                 "-DLLVM_BUILD_INSTRUMENTED=IR",
                 "-DLLVM_BUILD_RUNTIME=No",
             ]
-        if pgo_phase == "use":
+        elif profile:
             cmake_args += [
-                "-DLLVM_PROFDATA_FILE=%s/merged.profdata" % stage_dir,
+                "-DLLVM_PROFDATA_FILE=%s" % profile,
             ]
         return cmake_args
 
@@ -469,7 +469,7 @@ def prune_final_dir_for_clang_tidy(final_dir, osx_cross_compile):
             delete(f)
 
 
-if __name__ == "__main__":
+def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "-c",
@@ -573,8 +573,6 @@ if __name__ == "__main__":
         pgo = config["pgo"]
         if pgo not in (True, False):
             raise ValueError("Only boolean values are accepted for pgo.")
-        if pgo and stages != 4:
-            raise ValueError("PGO is only supported in 4-stage builds.")
     build_type = "Release"
     if "build_type" in config:
         build_type = config["build_type"]
@@ -787,7 +785,6 @@ if __name__ == "__main__":
         stage2_inst_dir = stage2_dir + "/" + package_name
         final_stage_dir = stage2_dir
         final_inst_dir = stage2_inst_dir
-        pgo_phase = "gen" if pgo else None
         if skip_stages < 1:
             cc = stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
             cxx = stage1_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
@@ -810,7 +807,7 @@ if __name__ == "__main__":
             libcxx_include_dir,
             build_wasm,
             is_final_stage=(stages == 2),
-            pgo_phase=pgo_phase,
+            profile="gen" if pgo else None,
         )
 
     if stages >= 3 and skip_stages < 3:
@@ -841,23 +838,30 @@ if __name__ == "__main__":
             build_wasm,
             (stages == 3),
         )
+        if pgo:
+            llvm_profdata = stage2_inst_dir + "/bin/llvm-profdata%s" % exe_ext
+            merge_cmd = [llvm_profdata, "merge", "-o", "merged.profdata"]
+            profraw_files = glob.glob(
+                os.path.join(stage2_dir, "build", "profiles", "*.profraw")
+            )
+            run_in(stage3_dir, merge_cmd + profraw_files)
+            if stages == 3:
+                mkdir_p(upload_dir)
+                shutil.copy2(os.path.join(stage3_dir, "merged.profdata"), upload_dir)
+                return
 
     if stages >= 4 and skip_stages < 4:
         stage4_dir = build_dir + "/stage4"
         stage4_inst_dir = stage4_dir + "/" + package_name
         final_stage_dir = stage4_dir
         final_inst_dir = stage4_inst_dir
-        pgo_phase = None
+        profile = None
         if pgo:
-            pgo_phase = "use"
-            llvm_profdata = stage3_inst_dir + "/bin/llvm-profdata%s" % exe_ext
-            merge_cmd = [llvm_profdata, "merge", "-o", "merged.profdata"]
-            profraw_files = glob.glob(
-                os.path.join(stage2_dir, "build", "profiles", "*.profraw")
-            )
-            if not os.path.exists(stage4_dir):
-                os.mkdir(stage4_dir)
-            run_in(stage4_dir, merge_cmd + profraw_files)
+            if skip_stages == 3:
+                profile_dir = os.environ.get("MOZ_FETCHES_DIR", "")
+            else:
+                profile_dir = stage3_dir
+            profile = os.path.join(profile_dir, "merged.profdata")
         if skip_stages < 3:
             cc = stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
             cxx = stage3_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
@@ -880,7 +884,7 @@ if __name__ == "__main__":
             libcxx_include_dir,
             build_wasm,
             (stages == 4),
-            pgo_phase=pgo_phase,
+            profile=profile,
         )
 
     if build_clang_tidy:
@@ -905,3 +909,7 @@ if __name__ == "__main__":
 
     if not args.skip_tar:
         build_tar_package("%s.tar.zst" % package_name, final_stage_dir, package_name)
+
+
+if __name__ == "__main__":
+    main()