Bug 1966470 - Part 1: Add mach artifact install --unfiltered-project-package command. r=firefox-build-system-reviewers,glandium

Getting artifacts from other locations when building is a solved
problem.  But single-locale repacks do something very different that
is very hard to arrange locally.  This commit will be used to make
single-locale repacks easier to work with.

This commit adds a new "unfiltered project package" mode that uses the
artifact build mode fetching code to download (and minimally process)
just the main package from a build.  The processing is convenient on
macOS, where DMG files are onerous to work with and the application
bundle name varies.

Differential Revision: https://phabricator.services.mozilla.com/D249447

python/mozbuild/mozbuild/artifact_commands.py

@@ -89,6 +89,7 @@ def _make_artifacts(
     download_symbols=False,
     download_maven_zip=False,
     no_process=False,
+    unfiltered_project_package=False,
 ):
     state_dir = command_context._mach_context.state_dir
     cache_dir = os.path.join(state_dir, "package-frontend")
@@ -129,6 +130,7 @@ def _make_artifacts(
         download_symbols=download_symbols,
         download_maven_zip=download_maven_zip,
         no_process=no_process,
+        unfiltered_project_package=unfiltered_project_package,
         mozbuild=command_context,
     )
     return artifacts
@@ -163,6 +165,11 @@ def _make_artifacts(
     action="store_true",
     help="Don't process (unpack) artifact packages, just download them.",
 )
+@CommandArgument(
+    "--unfiltered-project-package",
+    action="store_true",
+    help="Minimally process (only) main project package artifact, unpacking it to the given `--distdir`.",
+)
 @CommandArgument(
     "--maven-zip", action="store_true", help="Download Maven zip (Android-only)."
 )
@@ -177,6 +184,7 @@ def artifact_install(
     symbols=False,
     distdir=None,
     no_process=False,
+    unfiltered_project_package=False,
     maven_zip=False,
 ):
     command_context._set_log_level(verbose)
@@ -189,6 +197,7 @@ def artifact_install(
         download_symbols=symbols,
         download_maven_zip=maven_zip,
         no_process=no_process,
+        unfiltered_project_package=unfiltered_project_package,
     )
 
     return artifacts.install_from(source, distdir or command_context.distdir)

python/mozbuild/mozbuild/artifacts.py

@@ -57,7 +57,7 @@ import pylru
 import requests
 from mach.util import UserError
 from mozpack import executables
-from mozpack.files import JarFinder, TarFinder
+from mozpack.files import FileFinder, JarFinder, TarFinder
 from mozpack.mozjar import JarReader, JarWriter
 from mozpack.packager.unpack import UnpackFinder
 from taskgraph.util.taskcluster import find_task_id, get_artifact_url, list_artifacts
@@ -84,6 +84,9 @@ MAX_CACHED_TASKS = 400  # Number of pushheads to cache Task Cluster task data fo
 # copying from DMG files is very slow, we extract the desired binaries to a
 # separate archive for fast re-installation.
 PROCESSED_SUFFIX = ".processed.jar"
+UNFILTERED_PROJECT_PACKAGE_PROCESSED_SUFFIX = (
+    ".unfiltered_project_package.processed.jar"
+)
 
 
 class ArtifactJob:
@@ -861,6 +864,80 @@ class WinThunderbirdArtifactJob(ThunderbirdMixin, WinArtifactJob):
     pass
 
 
+class UnfilteredProjectPackageArtifactJob(ArtifactJob):
+    """An `ArtifactJob` that processes only the main project package and is
+    unfiltered, i.e., does not change the internal structure of the main
+    package.  For use in repackaging, where the artifact build mode VCS and
+    Taskcluster integration is convenient but the whole package is needed (and
+    DMGs are slow to work with locally).
+
+    Desktop-only at this time.
+
+    """
+
+    # Can't yet handle `AndroidArtifactJob` uniformly, since the `product` is "mobile".
+    package_re = "|".join(
+        [
+            f"({cls.package_re})"
+            for cls in (LinuxArtifactJob, MacArtifactJob, WinArtifactJob)
+        ]
+    )
+    product = "firefox"
+
+    @property
+    def _extra_archives(self):
+        return {}
+
+    def process_package_artifact(self, filename, processed_filename):
+        tempdir = tempfile.mkdtemp()
+        try:
+            self.log(
+                logging.DEBUG,
+                "artifact",
+                {"tempdir": tempdir},
+                "Unpacking into {tempdir}",
+            )
+            mozinstall.install(filename, tempdir)
+
+            # Avoid mismatches between local packages (Nightly.app) and CI artifacts
+            # (Firefox Nightly.app).
+            if filename.endswith(".dmg"):
+                bundle_dirs = glob.glob(mozpath.join(tempdir, "*.app"))
+            else:
+                bundle_dirs = glob.glob(
+                    mozpath.join(tempdir, self._substs["MOZ_APP_NAME"])
+                )
+
+            if len(bundle_dirs) != 1:
+                raise ValueError(f"Expected one source bundle, found: {bundle_dirs}")
+            (source,) = bundle_dirs
+
+            with self.get_writer(file=processed_filename, compress_level=5) as writer:
+                finder = FileFinder(source)
+                for p, f in finder.find("*"):
+                    q = p
+                    if filename.endswith(".dmg"):
+                        q = mozpath.join(self._substs["MOZ_MACBUNDLE_NAME"], q)
+                    self.log(
+                        logging.DEBUG,
+                        "artifact",
+                        {"path": q},
+                        "Adding {path} to unfiltered project package archive",
+                    )
+                    writer.add(q.encode("utf-8"), f.open(), mode=f.mode)
+
+        finally:
+            try:
+                shutil.rmtree(tempdir)
+            except OSError:
+                self.log(
+                    logging.WARN,
+                    "artifact",
+                    {"tempdir": tempdir},
+                    "Unable to delete {tempdir}",
+                )
+
+
 def startswithwhich(s, prefixes):
     for prefix in prefixes:
         if s.startswith(prefix):
@@ -1097,11 +1174,17 @@ class Artifacts:
         download_symbols=False,
         download_maven_zip=False,
         no_process=False,
+        unfiltered_project_package=False,
         mozbuild=None,
     ):
         if (hg and git) or (not hg and not git):
             raise ValueError("Must provide path to exactly one of hg and git")
 
+        if no_process and unfiltered_project_package:
+            raise ValueError(
+                "Must provide only one of no_process and unfiltered_project_package"
+            )
+
         self._substs = substs
         self._defines = defines
         self._tree = tree
@@ -1113,23 +1196,36 @@ class Artifacts:
         self._skip_cache = skip_cache
         self._topsrcdir = topsrcdir
         self._no_process = no_process
+        self._unfiltered_project_package = unfiltered_project_package
 
         app = self._substs.get("MOZ_BUILD_APP")
         job_details = COMM_JOB_DETAILS if app == "comm/mail" else MOZ_JOB_DETAILS
 
-        try:
+        if not self._unfiltered_project_package:
-            cls = job_details[self._job]
+            try:
-            self._artifact_job = cls(
+                cls = job_details[self._job]
+                self._artifact_job = cls(
+                    log=self._log,
+                    download_tests=download_tests,
+                    download_symbols=download_symbols,
+                    download_maven_zip=download_maven_zip,
+                    substs=self._substs,
+                    mozbuild=mozbuild,
+                )
+            except KeyError:
+                self.log(
+                    logging.INFO, "artifact", {"job": self._job}, "Unknown job {job}"
+                )
+                raise KeyError("Unknown job")
+        else:
+            self._artifact_job = UnfilteredProjectPackageArtifactJob(
                 log=self._log,
-                download_tests=download_tests,
+                download_tests=False,
-                download_symbols=download_symbols,
+                download_symbols=False,
-                download_maven_zip=download_maven_zip,
+                download_maven_zip=False,
                 substs=self._substs,
                 mozbuild=mozbuild,
             )
-        except KeyError:
-            self.log(logging.INFO, "artifact", {"job": self._job}, "Unknown job {job}")
-            raise KeyError("Unknown job")
 
         self._task_cache = TaskCache(
             self._cache_dir, log=self._log, skip_cache=self._skip_cache
@@ -1482,6 +1578,8 @@ https://firefox-source-docs.mozilla.org/contributing/vcs/mercurial_bundles.html
 
         # Do we need to post-process?
         processed_filename = filename + PROCESSED_SUFFIX
+        if self._unfiltered_project_package:
+            processed_filename = filename + UNFILTERED_PROJECT_PACKAGE_PROCESSED_SUFFIX
 
         if self._skip_cache and os.path.exists(processed_filename):
             self.log(