25a0a17cae
This step removes all the dependencies of mach commands to having a MachCommandBase as the `self` by using the `command_context` argument instead. This also removes any remaining statefulness from those classes that implement mach commands, ultimately making it easier to move existing commands out of classes in a follow-up. Differential Revision: https://phabricator.services.mozilla.com/D118058
605 lines
22 KiB
Python
605 lines
22 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import six
|
|
|
|
from collections import OrderedDict
|
|
|
|
from mach.decorators import CommandArgument, CommandProvider, Command, SubCommand
|
|
from mozbuild.artifact_builds import JOB_CHOICES
|
|
from mozbuild.base import MachCommandBase, MachCommandConditions as conditions
|
|
from mozbuild.util import ensureParentDir
|
|
import mozversioncontrol
|
|
|
|
|
|
_COULD_NOT_FIND_ARTIFACTS_TEMPLATE = (
|
|
"ERROR!!!!!! Could not find artifacts for a toolchain build named "
|
|
"`{build}`. Local commits, dirty/stale files, and other changes in your "
|
|
"checkout may cause this error. Make sure you are on a fresh, current "
|
|
"checkout of mozilla-central. Beware that commands like `mach bootstrap` "
|
|
"and `mach artifact` are unlikely to work on any versions of the code "
|
|
"besides recent revisions of mozilla-central."
|
|
)
|
|
|
|
|
|
class SymbolsAction(argparse.Action):
|
|
def __call__(self, parser, namespace, values, option_string=None):
|
|
# If this function is called, it means the --symbols option was given,
|
|
# so we want to store the value `True` if no explicit value was given
|
|
# to the option.
|
|
setattr(namespace, self.dest, values or True)
|
|
|
|
|
|
class ArtifactSubCommand(SubCommand):
|
|
def __call__(self, func):
|
|
after = SubCommand.__call__(self, func)
|
|
args = [
|
|
CommandArgument("--tree", metavar="TREE", type=str, help="Firefox tree."),
|
|
CommandArgument(
|
|
"--job", metavar="JOB", choices=JOB_CHOICES, help="Build job."
|
|
),
|
|
CommandArgument(
|
|
"--verbose", "-v", action="store_true", help="Print verbose output."
|
|
),
|
|
]
|
|
for arg in args:
|
|
after = arg(after)
|
|
return after
|
|
|
|
|
|
@CommandProvider
|
|
class PackageFrontend(MachCommandBase):
|
|
"""Fetch and install binary artifacts from Mozilla automation."""
|
|
|
|
@Command(
|
|
"artifact",
|
|
category="post-build",
|
|
description="Use pre-built artifacts to build Firefox.",
|
|
)
|
|
def artifact(self, command_context):
|
|
"""Download, cache, and install pre-built binary artifacts to build Firefox.
|
|
|
|
Use |mach build| as normal to freshen your installed binary libraries:
|
|
artifact builds automatically download, cache, and install binary
|
|
artifacts from Mozilla automation, replacing whatever may be in your
|
|
object directory. Use |mach artifact last| to see what binary artifacts
|
|
were last used.
|
|
|
|
Never build libxul again!
|
|
|
|
"""
|
|
pass
|
|
|
|
def _make_artifacts(
|
|
self,
|
|
command_context,
|
|
tree=None,
|
|
job=None,
|
|
skip_cache=False,
|
|
download_tests=True,
|
|
download_symbols=False,
|
|
download_host_bins=False,
|
|
download_maven_zip=False,
|
|
no_process=False,
|
|
):
|
|
state_dir = command_context._mach_context.state_dir
|
|
cache_dir = os.path.join(state_dir, "package-frontend")
|
|
|
|
hg = None
|
|
if conditions.is_hg(command_context):
|
|
hg = command_context.substs["HG"]
|
|
|
|
git = None
|
|
if conditions.is_git(command_context):
|
|
git = command_context.substs["GIT"]
|
|
|
|
# If we're building Thunderbird, we should be checking for comm-central artifacts.
|
|
topsrcdir = command_context.substs.get(
|
|
"commtopsrcdir", command_context.topsrcdir
|
|
)
|
|
|
|
if download_maven_zip:
|
|
if download_tests:
|
|
raise ValueError("--maven-zip requires --no-tests")
|
|
if download_symbols:
|
|
raise ValueError("--maven-zip requires no --symbols")
|
|
if download_host_bins:
|
|
raise ValueError("--maven-zip requires no --host-bins")
|
|
if not no_process:
|
|
raise ValueError("--maven-zip requires --no-process")
|
|
|
|
from mozbuild.artifacts import Artifacts
|
|
|
|
artifacts = Artifacts(
|
|
tree,
|
|
command_context.substs,
|
|
command_context.defines,
|
|
job,
|
|
log=command_context.log,
|
|
cache_dir=cache_dir,
|
|
skip_cache=skip_cache,
|
|
hg=hg,
|
|
git=git,
|
|
topsrcdir=topsrcdir,
|
|
download_tests=download_tests,
|
|
download_symbols=download_symbols,
|
|
download_host_bins=download_host_bins,
|
|
download_maven_zip=download_maven_zip,
|
|
no_process=no_process,
|
|
mozbuild=command_context,
|
|
)
|
|
return artifacts
|
|
|
|
@ArtifactSubCommand("artifact", "install", "Install a good pre-built artifact.")
|
|
@CommandArgument(
|
|
"source",
|
|
metavar="SRC",
|
|
nargs="?",
|
|
type=str,
|
|
help="Where to fetch and install artifacts from. Can be omitted, in "
|
|
"which case the current hg repository is inspected; an hg revision; "
|
|
"a remote URL; or a local file.",
|
|
default=None,
|
|
)
|
|
@CommandArgument(
|
|
"--skip-cache",
|
|
action="store_true",
|
|
help="Skip all local caches to force re-fetching remote artifacts.",
|
|
default=False,
|
|
)
|
|
@CommandArgument("--no-tests", action="store_true", help="Don't install tests.")
|
|
@CommandArgument(
|
|
"--symbols", nargs="?", action=SymbolsAction, help="Download symbols."
|
|
)
|
|
@CommandArgument("--host-bins", action="store_true", help="Download host binaries.")
|
|
@CommandArgument("--distdir", help="Where to install artifacts to.")
|
|
@CommandArgument(
|
|
"--no-process",
|
|
action="store_true",
|
|
help="Don't process (unpack) artifact packages, just download them.",
|
|
)
|
|
@CommandArgument(
|
|
"--maven-zip", action="store_true", help="Download Maven zip (Android-only)."
|
|
)
|
|
def artifact_install(
|
|
self,
|
|
command_context,
|
|
source=None,
|
|
skip_cache=False,
|
|
tree=None,
|
|
job=None,
|
|
verbose=False,
|
|
no_tests=False,
|
|
symbols=False,
|
|
host_bins=False,
|
|
distdir=None,
|
|
no_process=False,
|
|
maven_zip=False,
|
|
):
|
|
command_context._set_log_level(verbose)
|
|
artifacts = self._make_artifacts(
|
|
command_context,
|
|
tree=tree,
|
|
job=job,
|
|
skip_cache=skip_cache,
|
|
download_tests=not no_tests,
|
|
download_symbols=symbols,
|
|
download_host_bins=host_bins,
|
|
download_maven_zip=maven_zip,
|
|
no_process=no_process,
|
|
)
|
|
|
|
return artifacts.install_from(source, distdir or command_context.distdir)
|
|
|
|
@ArtifactSubCommand(
|
|
"artifact",
|
|
"clear-cache",
|
|
"Delete local artifacts and reset local artifact cache.",
|
|
)
|
|
def artifact_clear_cache(self, command_context, tree=None, job=None, verbose=False):
|
|
command_context._set_log_level(verbose)
|
|
artifacts = self._make_artifacts(command_context, tree=tree, job=job)
|
|
artifacts.clear_cache()
|
|
return 0
|
|
|
|
@SubCommand("artifact", "toolchain")
|
|
@CommandArgument(
|
|
"--verbose", "-v", action="store_true", help="Print verbose output."
|
|
)
|
|
@CommandArgument(
|
|
"--cache-dir",
|
|
metavar="DIR",
|
|
help="Directory where to store the artifacts cache",
|
|
)
|
|
@CommandArgument(
|
|
"--skip-cache",
|
|
action="store_true",
|
|
help="Skip all local caches to force re-fetching remote artifacts.",
|
|
default=False,
|
|
)
|
|
@CommandArgument(
|
|
"--from-build",
|
|
metavar="BUILD",
|
|
nargs="+",
|
|
help="Download toolchains resulting from the given build(s); "
|
|
"BUILD is a name of a toolchain task, e.g. linux64-clang",
|
|
)
|
|
@CommandArgument(
|
|
"--tooltool-manifest",
|
|
metavar="MANIFEST",
|
|
help="Explicit tooltool manifest to process",
|
|
)
|
|
@CommandArgument(
|
|
"--no-unpack", action="store_true", help="Do not unpack any downloaded file"
|
|
)
|
|
@CommandArgument(
|
|
"--retry", type=int, default=4, help="Number of times to retry failed downloads"
|
|
)
|
|
@CommandArgument(
|
|
"--bootstrap",
|
|
action="store_true",
|
|
help="Whether this is being called from bootstrap. "
|
|
"This verifies the toolchain is annotated as a toolchain used for local development.",
|
|
)
|
|
@CommandArgument(
|
|
"--artifact-manifest",
|
|
metavar="FILE",
|
|
help="Store a manifest about the downloaded taskcluster artifacts",
|
|
)
|
|
def artifact_toolchain(
|
|
self,
|
|
command_context,
|
|
verbose=False,
|
|
cache_dir=None,
|
|
skip_cache=False,
|
|
from_build=(),
|
|
tooltool_manifest=None,
|
|
no_unpack=False,
|
|
retry=0,
|
|
bootstrap=False,
|
|
artifact_manifest=None,
|
|
):
|
|
"""Download, cache and install pre-built toolchains."""
|
|
from mozbuild.artifacts import ArtifactCache
|
|
from mozbuild.action.tooltool import FileRecord, open_manifest, unpack_file
|
|
import redo
|
|
import requests
|
|
import time
|
|
|
|
from taskgraph.util.taskcluster import get_artifact_url
|
|
|
|
start = time.time()
|
|
command_context._set_log_level(verbose)
|
|
# Normally, we'd use command_context.log_manager.enable_unstructured(),
|
|
# but that enables all logging, while we only really want tooltool's
|
|
# and it also makes structured log output twice.
|
|
# So we manually do what it does, and limit that to the tooltool
|
|
# logger.
|
|
if command_context.log_manager.terminal_handler:
|
|
logging.getLogger("mozbuild.action.tooltool").addHandler(
|
|
command_context.log_manager.terminal_handler
|
|
)
|
|
logging.getLogger("redo").addHandler(
|
|
command_context.log_manager.terminal_handler
|
|
)
|
|
command_context.log_manager.terminal_handler.addFilter(
|
|
command_context.log_manager.structured_filter
|
|
)
|
|
if not cache_dir:
|
|
cache_dir = os.path.join(
|
|
command_context._mach_context.state_dir, "toolchains"
|
|
)
|
|
|
|
tooltool_host = os.environ.get("TOOLTOOL_HOST", "tooltool.mozilla-releng.net")
|
|
taskcluster_proxy_url = os.environ.get("TASKCLUSTER_PROXY_URL")
|
|
if taskcluster_proxy_url:
|
|
tooltool_url = "{}/{}".format(taskcluster_proxy_url, tooltool_host)
|
|
else:
|
|
tooltool_url = "https://{}".format(tooltool_host)
|
|
|
|
cache = ArtifactCache(
|
|
cache_dir=cache_dir, log=command_context.log, skip_cache=skip_cache
|
|
)
|
|
|
|
class DownloadRecord(FileRecord):
|
|
def __init__(self, url, *args, **kwargs):
|
|
super(DownloadRecord, self).__init__(*args, **kwargs)
|
|
self.url = url
|
|
self.basename = self.filename
|
|
|
|
def fetch_with(self, cache):
|
|
self.filename = cache.fetch(self.url)
|
|
return self.filename
|
|
|
|
def validate(self):
|
|
if self.size is None and self.digest is None:
|
|
return True
|
|
return super(DownloadRecord, self).validate()
|
|
|
|
class ArtifactRecord(DownloadRecord):
|
|
def __init__(self, task_id, artifact_name):
|
|
for _ in redo.retrier(attempts=retry + 1, sleeptime=60):
|
|
cot = cache._download_manager.session.get(
|
|
get_artifact_url(task_id, "public/chain-of-trust.json")
|
|
)
|
|
if cot.status_code >= 500:
|
|
continue
|
|
cot.raise_for_status()
|
|
break
|
|
else:
|
|
cot.raise_for_status()
|
|
|
|
digest = algorithm = None
|
|
data = json.loads(cot.text)
|
|
for algorithm, digest in (
|
|
data.get("artifacts", {}).get(artifact_name, {}).items()
|
|
):
|
|
pass
|
|
|
|
name = os.path.basename(artifact_name)
|
|
artifact_url = get_artifact_url(
|
|
task_id,
|
|
artifact_name,
|
|
use_proxy=not artifact_name.startswith("public/"),
|
|
)
|
|
super(ArtifactRecord, self).__init__(
|
|
artifact_url, name, None, digest, algorithm, unpack=True
|
|
)
|
|
|
|
records = OrderedDict()
|
|
downloaded = []
|
|
|
|
if tooltool_manifest:
|
|
manifest = open_manifest(tooltool_manifest)
|
|
for record in manifest.file_records:
|
|
url = "{}/{}/{}".format(tooltool_url, record.algorithm, record.digest)
|
|
records[record.filename] = DownloadRecord(
|
|
url,
|
|
record.filename,
|
|
record.size,
|
|
record.digest,
|
|
record.algorithm,
|
|
unpack=record.unpack,
|
|
version=record.version,
|
|
visibility=record.visibility,
|
|
)
|
|
|
|
if from_build:
|
|
if "MOZ_AUTOMATION" in os.environ:
|
|
command_context.log(
|
|
logging.ERROR,
|
|
"artifact",
|
|
{},
|
|
"Do not use --from-build in automation; all dependencies "
|
|
"should be determined in the decision task.",
|
|
)
|
|
return 1
|
|
from taskgraph.optimize.strategies import IndexSearch
|
|
from mozbuild.toolchains import toolchain_task_definitions
|
|
|
|
tasks = toolchain_task_definitions()
|
|
|
|
for b in from_build:
|
|
user_value = b
|
|
|
|
if not b.startswith("toolchain-"):
|
|
b = "toolchain-{}".format(b)
|
|
|
|
task = tasks.get(b)
|
|
if not task:
|
|
command_context.log(
|
|
logging.ERROR,
|
|
"artifact",
|
|
{"build": user_value},
|
|
"Could not find a toolchain build named `{build}`",
|
|
)
|
|
return 1
|
|
|
|
# Ensure that toolchains installed by `mach bootstrap` have the
|
|
# `local-toolchain attribute set. Taskgraph ensures that these
|
|
# are built on trunk projects, so the task will be available to
|
|
# install here.
|
|
if bootstrap and not task.attributes.get("local-toolchain"):
|
|
command_context.log(
|
|
logging.ERROR,
|
|
"artifact",
|
|
{"build": user_value},
|
|
"Toolchain `{build}` is not annotated as used for local development.",
|
|
)
|
|
return 1
|
|
|
|
artifact_name = task.attributes.get("toolchain-artifact")
|
|
command_context.log(
|
|
logging.DEBUG,
|
|
"artifact",
|
|
{
|
|
"name": artifact_name,
|
|
"index": task.optimization.get("index-search"),
|
|
},
|
|
"Searching for {name} in {index}",
|
|
)
|
|
deadline = None
|
|
task_id = IndexSearch().should_replace_task(
|
|
task, {}, deadline, task.optimization.get("index-search", [])
|
|
)
|
|
if task_id in (True, False) or not artifact_name:
|
|
command_context.log(
|
|
logging.ERROR,
|
|
"artifact",
|
|
{"build": user_value},
|
|
_COULD_NOT_FIND_ARTIFACTS_TEMPLATE,
|
|
)
|
|
# Get and print some helpful info for diagnosis.
|
|
repo = mozversioncontrol.get_repository_object(
|
|
command_context.topsrcdir
|
|
)
|
|
changed_files = set(repo.get_outgoing_files()) | set(
|
|
repo.get_changed_files()
|
|
)
|
|
if changed_files:
|
|
command_context.log(
|
|
logging.ERROR,
|
|
"artifact",
|
|
{},
|
|
"Hint: consider reverting your local changes "
|
|
"to the following files: %s" % sorted(changed_files),
|
|
)
|
|
if "TASKCLUSTER_ROOT_URL" in os.environ:
|
|
command_context.log(
|
|
logging.ERROR,
|
|
"artifact",
|
|
{"build": user_value},
|
|
"Due to the environment variable TASKCLUSTER_ROOT_URL "
|
|
"being set, the artifacts were expected to be found "
|
|
"on {}. If this was unintended, unset "
|
|
"TASKCLUSTER_ROOT_URL and try again.".format(
|
|
os.environ["TASKCLUSTER_ROOT_URL"]
|
|
),
|
|
)
|
|
return 1
|
|
|
|
command_context.log(
|
|
logging.DEBUG,
|
|
"artifact",
|
|
{"name": artifact_name, "task_id": task_id},
|
|
"Found {name} in {task_id}",
|
|
)
|
|
|
|
record = ArtifactRecord(task_id, artifact_name)
|
|
records[record.filename] = record
|
|
|
|
for record in six.itervalues(records):
|
|
command_context.log(
|
|
logging.INFO,
|
|
"artifact",
|
|
{"name": record.basename},
|
|
"Setting up artifact {name}",
|
|
)
|
|
valid = False
|
|
# sleeptime is 60 per retry.py, used by tooltool_wrapper.sh
|
|
for attempt, _ in enumerate(redo.retrier(attempts=retry + 1, sleeptime=60)):
|
|
try:
|
|
record.fetch_with(cache)
|
|
except (
|
|
requests.exceptions.HTTPError,
|
|
requests.exceptions.ChunkedEncodingError,
|
|
requests.exceptions.ConnectionError,
|
|
) as e:
|
|
|
|
if isinstance(e, requests.exceptions.HTTPError):
|
|
# The relengapi proxy likes to return error 400 bad request
|
|
# which seems improbably to be due to our (simple) GET
|
|
# being borked.
|
|
status = e.response.status_code
|
|
should_retry = status >= 500 or status == 400
|
|
else:
|
|
should_retry = True
|
|
|
|
if should_retry or attempt < retry:
|
|
level = logging.WARN
|
|
else:
|
|
level = logging.ERROR
|
|
command_context.log(level, "artifact", {}, str(e))
|
|
if not should_retry:
|
|
break
|
|
if attempt < retry:
|
|
command_context.log(
|
|
logging.INFO, "artifact", {}, "Will retry in a moment..."
|
|
)
|
|
continue
|
|
try:
|
|
valid = record.validate()
|
|
except Exception:
|
|
pass
|
|
if not valid:
|
|
os.unlink(record.filename)
|
|
if attempt < retry:
|
|
command_context.log(
|
|
logging.INFO,
|
|
"artifact",
|
|
{},
|
|
"Corrupt download. Will retry in a moment...",
|
|
)
|
|
continue
|
|
|
|
downloaded.append(record)
|
|
break
|
|
|
|
if not valid:
|
|
command_context.log(
|
|
logging.ERROR,
|
|
"artifact",
|
|
{"name": record.basename},
|
|
"Failed to download {name}",
|
|
)
|
|
return 1
|
|
|
|
artifacts = {} if artifact_manifest else None
|
|
|
|
for record in downloaded:
|
|
local = os.path.join(os.getcwd(), record.basename)
|
|
if os.path.exists(local):
|
|
os.unlink(local)
|
|
# unpack_file needs the file with its final name to work
|
|
# (https://github.com/mozilla/build-tooltool/issues/38), so we
|
|
# need to copy it, even though we remove it later. Use hard links
|
|
# when possible.
|
|
try:
|
|
os.link(record.filename, local)
|
|
except Exception:
|
|
shutil.copy(record.filename, local)
|
|
# Keep a sha256 of each downloaded file, for the chain-of-trust
|
|
# validation.
|
|
if artifact_manifest is not None:
|
|
with open(local, "rb") as fh:
|
|
h = hashlib.sha256()
|
|
while True:
|
|
data = fh.read(1024 * 1024)
|
|
if not data:
|
|
break
|
|
h.update(data)
|
|
artifacts[record.url] = {"sha256": h.hexdigest()}
|
|
if record.unpack and not no_unpack:
|
|
unpack_file(local)
|
|
os.unlink(local)
|
|
|
|
if not downloaded:
|
|
command_context.log(logging.ERROR, "artifact", {}, "Nothing to download")
|
|
|
|
if artifacts:
|
|
ensureParentDir(artifact_manifest)
|
|
with open(artifact_manifest, "w") as fh:
|
|
json.dump(artifacts, fh, indent=4, sort_keys=True)
|
|
|
|
if "MOZ_AUTOMATION" in os.environ:
|
|
end = time.time()
|
|
|
|
perfherder_data = {
|
|
"framework": {"name": "build_metrics"},
|
|
"suites": [
|
|
{
|
|
"name": "mach_artifact_toolchain",
|
|
"value": end - start,
|
|
"lowerIsBetter": True,
|
|
"shouldAlert": False,
|
|
"subtests": [],
|
|
}
|
|
],
|
|
}
|
|
command_context.log(
|
|
logging.INFO,
|
|
"perfherder",
|
|
{"data": json.dumps(perfherder_data)},
|
|
"PERFHERDER_DATA: {data}",
|
|
)
|
|
|
|
return 0
|