corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1966470 - Part 1: Add mach artifact install --unfiltered-project-package command. r=firefox-build-system-reviewers,glandium

Browse Source 
Getting artifacts from other locations when building is a solved
problem.  But single-locale repacks do something very different that
is very hard to arrange locally.  This commit will be used to make
single-locale repacks easier to work with.

This commit adds a new "unfiltered project package" mode that uses the
artifact build mode fetching code to download (and minimally process)
just the main package from a build.  The processing is convenient on
macOS, where DMG files are onerous to work with and the application
bundle name varies.

Differential Revision: https://phabricator.services.mozilla.com/D249447
This commit is contained in:
Nick Alexander
2025-05-21 16:32:56 +00:00
committed by nalexander@mozilla.com
parent f4d2d352e2
commit a8d08169d9
2 changed files with 117 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
python/mozbuild/mozbuild/artifact_commands.py
View File

@@ -89,6 +89,7 @@ def _make_artifacts(
download_symbols=False,
download_maven_zip=False,
no_process=False,
unfiltered_project_package=False,
):
state_dir = command_context._mach_context.state_dir
cache_dir = os.path.join(state_dir, "package-frontend")
@@ -129,6 +130,7 @@ def _make_artifacts(
download_symbols=download_symbols,
download_maven_zip=download_maven_zip,
no_process=no_process,
unfiltered_project_package=unfiltered_project_package,
mozbuild=command_context,
)
return artifacts
@@ -163,6 +165,11 @@ def _make_artifacts(
action="store_true",
help="Don't process (unpack) artifact packages, just download them.",
)
@CommandArgument(
"--unfiltered-project-package",
action="store_true",
help="Minimally process (only) main project package artifact, unpacking it to the given `--distdir`.",
)
@CommandArgument(
"--maven-zip", action="store_true", help="Download Maven zip (Android-only)."
)
@@ -177,6 +184,7 @@ def artifact_install(
symbols=False,
distdir=None,
no_process=False,
unfiltered_project_package=False,
maven_zip=False,
):
command_context._set_log_level(verbose)
@@ -189,6 +197,7 @@ def artifact_install(
download_symbols=symbols,
download_maven_zip=maven_zip,
no_process=no_process,
unfiltered_project_package=unfiltered_project_package,
)
return artifacts.install_from(source, distdir or command_context.distdir)

118
python/mozbuild/mozbuild/artifacts.py
View File

@@ -57,7 +57,7 @@ import pylru
import requests
from mach.util import UserError
from mozpack import executables
from mozpack.files import JarFinder, TarFinder
from mozpack.files import FileFinder, JarFinder, TarFinder
from mozpack.mozjar import JarReader, JarWriter
from mozpack.packager.unpack import UnpackFinder
from taskgraph.util.taskcluster import find_task_id, get_artifact_url, list_artifacts
@@ -84,6 +84,9 @@ MAX_CACHED_TASKS = 400 # Number of pushheads to cache Task Cluster task data fo
# copying from DMG files is very slow, we extract the desired binaries to a
# separate archive for fast re-installation.
PROCESSED_SUFFIX = ".processed.jar"
UNFILTERED_PROJECT_PACKAGE_PROCESSED_SUFFIX = (
".unfiltered_project_package.processed.jar"
)
class ArtifactJob:
@@ -861,6 +864,80 @@ class WinThunderbirdArtifactJob(ThunderbirdMixin, WinArtifactJob):
pass
class UnfilteredProjectPackageArtifactJob(ArtifactJob):
"""An `ArtifactJob` that processes only the main project package and is
unfiltered, i.e., does not change the internal structure of the main
package. For use in repackaging, where the artifact build mode VCS and
Taskcluster integration is convenient but the whole package is needed (and
DMGs are slow to work with locally).
Desktop-only at this time.
"""
# Can't yet handle `AndroidArtifactJob` uniformly, since the `product` is "mobile".
package_re = "|".join(
[
f"({cls.package_re})"
for cls in (LinuxArtifactJob, MacArtifactJob, WinArtifactJob)
]
)
product = "firefox"
@property
def _extra_archives(self):
return {}
def process_package_artifact(self, filename, processed_filename):
tempdir = tempfile.mkdtemp()
try:
self.log(
logging.DEBUG,
"artifact",
{"tempdir": tempdir},
"Unpacking into {tempdir}",
)
mozinstall.install(filename, tempdir)
# Avoid mismatches between local packages (Nightly.app) and CI artifacts
# (Firefox Nightly.app).
if filename.endswith(".dmg"):
bundle_dirs = glob.glob(mozpath.join(tempdir, "*.app"))
else:
bundle_dirs = glob.glob(
mozpath.join(tempdir, self._substs["MOZ_APP_NAME"])
)
if len(bundle_dirs) != 1:
raise ValueError(f"Expected one source bundle, found: {bundle_dirs}")
(source,) = bundle_dirs
with self.get_writer(file=processed_filename, compress_level=5) as writer:
finder = FileFinder(source)
for p, f in finder.find("*"):
q = p
if filename.endswith(".dmg"):
q = mozpath.join(self._substs["MOZ_MACBUNDLE_NAME"], q)
self.log(
logging.DEBUG,
"artifact",
{"path": q},
"Adding {path} to unfiltered project package archive",
)
writer.add(q.encode("utf-8"), f.open(), mode=f.mode)
finally:
try:
shutil.rmtree(tempdir)
except OSError:
self.log(
logging.WARN,
"artifact",
{"tempdir": tempdir},
"Unable to delete {tempdir}",
)
def startswithwhich(s, prefixes):
for prefix in prefixes:
if s.startswith(prefix):
@@ -1097,11 +1174,17 @@ class Artifacts:
download_symbols=False,
download_maven_zip=False,
no_process=False,
unfiltered_project_package=False,
mozbuild=None,
):
if (hg and git) or (not hg and not git):
raise ValueError("Must provide path to exactly one of hg and git")
if no_process and unfiltered_project_package:
raise ValueError(
"Must provide only one of no_process and unfiltered_project_package"
)
self._substs = substs
self._defines = defines
self._tree = tree
@@ -1113,23 +1196,36 @@ class Artifacts:
self._skip_cache = skip_cache
self._topsrcdir = topsrcdir
self._no_process = no_process
self._unfiltered_project_package = unfiltered_project_package
app = self._substs.get("MOZ_BUILD_APP")
job_details = COMM_JOB_DETAILS if app == "comm/mail" else MOZ_JOB_DETAILS
try:
cls = job_details[self._job]
self._artifact_job = cls(
if not self._unfiltered_project_package:
try:
cls = job_details[self._job]
self._artifact_job = cls(
log=self._log,
download_tests=download_tests,
download_symbols=download_symbols,
download_maven_zip=download_maven_zip,
substs=self._substs,
mozbuild=mozbuild,
)
except KeyError:
self.log(
logging.INFO, "artifact", {"job": self._job}, "Unknown job {job}"
)
raise KeyError("Unknown job")
else:
self._artifact_job = UnfilteredProjectPackageArtifactJob(
log=self._log,
download_tests=download_tests,
download_symbols=download_symbols,
download_maven_zip=download_maven_zip,
download_tests=False,
download_symbols=False,
download_maven_zip=False,
substs=self._substs,
mozbuild=mozbuild,
)
except KeyError:
self.log(logging.INFO, "artifact", {"job": self._job}, "Unknown job {job}")
raise KeyError("Unknown job")
self._task_cache = TaskCache(
self._cache_dir, log=self._log, skip_cache=self._skip_cache
@@ -1482,6 +1578,8 @@ https://firefox-source-docs.mozilla.org/contributing/vcs/mercurial_bundles.html
# Do we need to post-process?
processed_filename = filename + PROCESSED_SUFFIX
if self._unfiltered_project_package:
processed_filename = filename + UNFILTERED_PROJECT_PACKAGE_PROCESSED_SUFFIX
if self._skip_cache and os.path.exists(processed_filename):
self.log(