From a8d08169d9fd305d40783410b2960cd7816145d0 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Wed, 21 May 2025 16:32:56 +0000 Subject: [PATCH] Bug 1966470 - Part 1: Add `mach artifact install --unfiltered-project-package` command. r=firefox-build-system-reviewers,glandium Getting artifacts from other locations when building is a solved problem. But single-locale repacks do something very different that is very hard to arrange locally. This commit will be used to make single-locale repacks easier to work with. This commit adds a new "unfiltered project package" mode that uses the artifact build mode fetching code to download (and minimally process) just the main package from a build. The processing is convenient on macOS, where DMG files are onerous to work with and the application bundle name varies. Differential Revision: https://phabricator.services.mozilla.com/D249447 --- python/mozbuild/mozbuild/artifact_commands.py | 9 ++ python/mozbuild/mozbuild/artifacts.py | 118 ++++++++++++++++-- 2 files changed, 117 insertions(+), 10 deletions(-) diff --git a/python/mozbuild/mozbuild/artifact_commands.py b/python/mozbuild/mozbuild/artifact_commands.py index 83da36b3940b..1951a41546c7 100644 --- a/python/mozbuild/mozbuild/artifact_commands.py +++ b/python/mozbuild/mozbuild/artifact_commands.py @@ -89,6 +89,7 @@ def _make_artifacts( download_symbols=False, download_maven_zip=False, no_process=False, + unfiltered_project_package=False, ): state_dir = command_context._mach_context.state_dir cache_dir = os.path.join(state_dir, "package-frontend") @@ -129,6 +130,7 @@ def _make_artifacts( download_symbols=download_symbols, download_maven_zip=download_maven_zip, no_process=no_process, + unfiltered_project_package=unfiltered_project_package, mozbuild=command_context, ) return artifacts @@ -163,6 +165,11 @@ def _make_artifacts( action="store_true", help="Don't process (unpack) artifact packages, just download them.", ) +@CommandArgument( + "--unfiltered-project-package", + action="store_true", + help="Minimally process (only) main project package artifact, unpacking it to the given `--distdir`.", +) @CommandArgument( "--maven-zip", action="store_true", help="Download Maven zip (Android-only)." ) @@ -177,6 +184,7 @@ def artifact_install( symbols=False, distdir=None, no_process=False, + unfiltered_project_package=False, maven_zip=False, ): command_context._set_log_level(verbose) @@ -189,6 +197,7 @@ def artifact_install( download_symbols=symbols, download_maven_zip=maven_zip, no_process=no_process, + unfiltered_project_package=unfiltered_project_package, ) return artifacts.install_from(source, distdir or command_context.distdir) diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py index 60b483bf1d8c..e6348978352f 100644 --- a/python/mozbuild/mozbuild/artifacts.py +++ b/python/mozbuild/mozbuild/artifacts.py @@ -57,7 +57,7 @@ import pylru import requests from mach.util import UserError from mozpack import executables -from mozpack.files import JarFinder, TarFinder +from mozpack.files import FileFinder, JarFinder, TarFinder from mozpack.mozjar import JarReader, JarWriter from mozpack.packager.unpack import UnpackFinder from taskgraph.util.taskcluster import find_task_id, get_artifact_url, list_artifacts @@ -84,6 +84,9 @@ MAX_CACHED_TASKS = 400 # Number of pushheads to cache Task Cluster task data fo # copying from DMG files is very slow, we extract the desired binaries to a # separate archive for fast re-installation. PROCESSED_SUFFIX = ".processed.jar" +UNFILTERED_PROJECT_PACKAGE_PROCESSED_SUFFIX = ( + ".unfiltered_project_package.processed.jar" +) class ArtifactJob: @@ -861,6 +864,80 @@ class WinThunderbirdArtifactJob(ThunderbirdMixin, WinArtifactJob): pass +class UnfilteredProjectPackageArtifactJob(ArtifactJob): + """An `ArtifactJob` that processes only the main project package and is + unfiltered, i.e., does not change the internal structure of the main + package. For use in repackaging, where the artifact build mode VCS and + Taskcluster integration is convenient but the whole package is needed (and + DMGs are slow to work with locally). + + Desktop-only at this time. + + """ + + # Can't yet handle `AndroidArtifactJob` uniformly, since the `product` is "mobile". + package_re = "|".join( + [ + f"({cls.package_re})" + for cls in (LinuxArtifactJob, MacArtifactJob, WinArtifactJob) + ] + ) + product = "firefox" + + @property + def _extra_archives(self): + return {} + + def process_package_artifact(self, filename, processed_filename): + tempdir = tempfile.mkdtemp() + try: + self.log( + logging.DEBUG, + "artifact", + {"tempdir": tempdir}, + "Unpacking into {tempdir}", + ) + mozinstall.install(filename, tempdir) + + # Avoid mismatches between local packages (Nightly.app) and CI artifacts + # (Firefox Nightly.app). + if filename.endswith(".dmg"): + bundle_dirs = glob.glob(mozpath.join(tempdir, "*.app")) + else: + bundle_dirs = glob.glob( + mozpath.join(tempdir, self._substs["MOZ_APP_NAME"]) + ) + + if len(bundle_dirs) != 1: + raise ValueError(f"Expected one source bundle, found: {bundle_dirs}") + (source,) = bundle_dirs + + with self.get_writer(file=processed_filename, compress_level=5) as writer: + finder = FileFinder(source) + for p, f in finder.find("*"): + q = p + if filename.endswith(".dmg"): + q = mozpath.join(self._substs["MOZ_MACBUNDLE_NAME"], q) + self.log( + logging.DEBUG, + "artifact", + {"path": q}, + "Adding {path} to unfiltered project package archive", + ) + writer.add(q.encode("utf-8"), f.open(), mode=f.mode) + + finally: + try: + shutil.rmtree(tempdir) + except OSError: + self.log( + logging.WARN, + "artifact", + {"tempdir": tempdir}, + "Unable to delete {tempdir}", + ) + + def startswithwhich(s, prefixes): for prefix in prefixes: if s.startswith(prefix): @@ -1097,11 +1174,17 @@ class Artifacts: download_symbols=False, download_maven_zip=False, no_process=False, + unfiltered_project_package=False, mozbuild=None, ): if (hg and git) or (not hg and not git): raise ValueError("Must provide path to exactly one of hg and git") + if no_process and unfiltered_project_package: + raise ValueError( + "Must provide only one of no_process and unfiltered_project_package" + ) + self._substs = substs self._defines = defines self._tree = tree @@ -1113,23 +1196,36 @@ class Artifacts: self._skip_cache = skip_cache self._topsrcdir = topsrcdir self._no_process = no_process + self._unfiltered_project_package = unfiltered_project_package app = self._substs.get("MOZ_BUILD_APP") job_details = COMM_JOB_DETAILS if app == "comm/mail" else MOZ_JOB_DETAILS - try: - cls = job_details[self._job] - self._artifact_job = cls( + if not self._unfiltered_project_package: + try: + cls = job_details[self._job] + self._artifact_job = cls( + log=self._log, + download_tests=download_tests, + download_symbols=download_symbols, + download_maven_zip=download_maven_zip, + substs=self._substs, + mozbuild=mozbuild, + ) + except KeyError: + self.log( + logging.INFO, "artifact", {"job": self._job}, "Unknown job {job}" + ) + raise KeyError("Unknown job") + else: + self._artifact_job = UnfilteredProjectPackageArtifactJob( log=self._log, - download_tests=download_tests, - download_symbols=download_symbols, - download_maven_zip=download_maven_zip, + download_tests=False, + download_symbols=False, + download_maven_zip=False, substs=self._substs, mozbuild=mozbuild, ) - except KeyError: - self.log(logging.INFO, "artifact", {"job": self._job}, "Unknown job {job}") - raise KeyError("Unknown job") self._task_cache = TaskCache( self._cache_dir, log=self._log, skip_cache=self._skip_cache @@ -1482,6 +1578,8 @@ https://firefox-source-docs.mozilla.org/contributing/vcs/mercurial_bundles.html # Do we need to post-process? processed_filename = filename + PROCESSED_SUFFIX + if self._unfiltered_project_package: + processed_filename = filename + UNFILTERED_PROJECT_PACKAGE_PROCESSED_SUFFIX if self._skip_cache and os.path.exists(processed_filename): self.log(