From fdc33dac57ea91903d23b5ce01d84c47089253b4 Mon Sep 17 00:00:00 2001 From: Mitchell Hentges Date: Thu, 6 Jan 2022 00:32:48 +0000 Subject: [PATCH] Bug 1725895: Port `--profile-command` to pure-Python r=nalexander,glandium As part of this, the shell-script part of `./mach` can be removed, making it pure Python. There's a change in `--profile-command` behaviour, though: it now only profiles the specific command, rather than all of Mach. This is because _so much of Mach_ has already been run before CLI arguments are parsed in the Python process. If a developer wants to profile Mach itself, they can manually run `python3 -m cProfile -o ./mach ...` Differential Revision: https://phabricator.services.mozilla.com/D133928 --- build/moz.configure/bootstrap.configure | 6 +- js/src/devtools/automation/autospider.py | 3 +- mach | 76 +------------------ python/mach/docs/faq.rst | 29 +++++-- python/mach/mach/main.py | 1 + python/mach/mach/registrar.py | 24 +++++- .../mozharness/mozilla/building/buildbase.py | 14 +--- 7 files changed, 52 insertions(+), 101 deletions(-) diff --git a/build/moz.configure/bootstrap.configure b/build/moz.configure/bootstrap.configure index 1ed517fed05e..de8f76d52c62 100755 --- a/build/moz.configure/bootstrap.configure +++ b/build/moz.configure/bootstrap.configure @@ -116,17 +116,17 @@ def bootstrap_path(path, **kwargs): "--enable-bootstrap", toolchains_base_dir, bootstrap_toolchain_tasks, - shell, build_environment, dependable(path), when=when, ) @imports("os") @imports("subprocess") + @imports("sys") @imports(_from="mozbuild.util", _import="ensureParentDir") @imports(_from="__builtin__", _import="open") @imports(_from="__builtin__", _import="Exception") - def bootstrap_path(bootstrap, toolchains_base_dir, tasks, shell, build_env, path): + def bootstrap_path(bootstrap, toolchains_base_dir, tasks, build_env, path): path_parts = path.split("/") def try_bootstrap(exists): @@ -169,7 +169,7 @@ def bootstrap_path(path, **kwargs): os.makedirs(toolchains_base_dir, exist_ok=True) subprocess.run( [ - shell, + sys.executable, os.path.join(build_env.topsrcdir, "mach"), "--log-no-times", "artifact", diff --git a/js/src/devtools/automation/autospider.py b/js/src/devtools/automation/autospider.py index da3bb6b493a7..27f5c8f9da6a 100755 --- a/js/src/devtools/automation/autospider.py +++ b/js/src/devtools/automation/autospider.py @@ -470,7 +470,7 @@ mach = posixpath.join(PDIR.source, "mach") if not args.nobuild: # Do the build - run_command([mach, "build"], check=True) + run_command([sys.executable, mach, "build"], check=True) if use_minidump: # Convert symbols to breakpad format. @@ -481,6 +481,7 @@ if not args.nobuild: cmd_env["MOZ_AUTOMATION_BUILD_SYMBOLS"] = "1" run_command( [ + sys.executable, mach, "build", "recurse_syms", diff --git a/mach b/mach index 062772793186..813dcdc99d59 100755 --- a/mach +++ b/mach @@ -1,82 +1,8 @@ -#!/bin/sh +#!/usr/bin/env python3 # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -# The beginning of this script is both valid POSIX shell and valid Python, -# such that the script starts with the shell and is reexecuted with -# the right Python. - -# Embeds a shell script inside a Python triple quote. This pattern is valid -# shell because `''':'`, `':'` and `:` are all equivalent, and `:` is a no-op. -''':' - -get_command() { - # Parse the name of the mach command out of the arguments. This is necessary - # in the presence of global mach arguments that come before the name of the - # command, e.g. `mach -v build`. We dispatch to the correct Python - # interpreter depending on the command. - while true; do - case $1 in - -v|--verbose) shift;; - -l|--log-file) - if [ "$#" -lt 2 ] - then - echo - break - else - shift 2 - fi - ;; - --no-interactive) shift;; - --log-interval) shift;; - --log-no-times) shift;; - -h) shift;; - --debug-command) shift;; - --profile-command) - py_profile_command="1" - shift;; - --settings) - if [ "$#" -lt 2 ] - then - echo - break - else - shift 2 - fi - ;; - "") echo; break;; - *) echo $1; break;; - esac - done - return ${py_profile_command} -} - -command=$(get_command "$@") -py_profile_command=$? - -if [ ${py_profile_command} -eq 0 ] -then - py_profile_command_args="" -else - # We would prefer to use an array variable here, but we're limited to POSIX. - # None of our arguments have quoting or spaces so we can safely interpolate - # a string instead. - py_profile_command_args="-m cProfile -o mach_profile_${command}.cProfile" - echo "Running with --profile-command. To visualize, use snakeviz:" - echo "python3 -m pip install snakeviz" - echo "python3 -m snakeviz mach_profile_${command}.cProfile" -fi - -if command -v python3 > /dev/null -then - exec python3 $py_profile_command_args "$0" "$@" -else - echo "This mach command requires 'python3', which wasn't found on the system!" - exit 1 -fi -''' - from __future__ import absolute_import, print_function, unicode_literals import os diff --git a/python/mach/docs/faq.rst b/python/mach/docs/faq.rst index ee9be3daacdf..6413e8b44da6 100644 --- a/python/mach/docs/faq.rst +++ b/python/mach/docs/faq.rst @@ -33,18 +33,31 @@ when the command is invoked with: How do I profile a slow command? -------------------------------- -You can run a command and capture a profile as the ``mach`` process -loads and invokes the command with: +To diagnose bottlenecks, you can collect a performance profile: .. code-block:: shell - ./mach --profile-command SLOW-COMMAND ARGS ... + ./mach --profile-command SLOW-COMMAND ARGS ... -Look for a ``mach_profile_SLOW-COMMAND.cProfile`` file. You can -visualize using `snakeviz `__. -Instructions on how to install and use ``snakeviz`` are printed to the -console, since it can be tricky to target the correct Python virtual -environment. +Then, you can visualize ``mach_profile_SLOW-COMMAND.cProfile`` using +`snakeviz `__: + +.. code-block:: shell + + # If you don't have snakeviz installed yet: + python3 -m pip install snakeviz + python3 -m snakeviz mach_profile_SLOW-COMMAND.cProfile + +How do I profile ``mach`` itself? +--------------------------------- + +Since ``--profile-command`` only profiles commands, you'll need to invoke ``cProfile`` +directly to profile ``mach`` itself: + +.. code-block:: shell + + python3 -m cProfile -o mach.cProfile ./mach ... + python3 -m snakeviz mach.cProfile Is ``mach`` a build system? --------------------------- diff --git a/python/mach/mach/main.py b/python/mach/mach/main.py index a8a525811c3d..c2bb7d7b0edd 100644 --- a/python/mach/mach/main.py +++ b/python/mach/mach/main.py @@ -490,6 +490,7 @@ To see more help for a specific command, run: handler, context, debug_command=args.debug_command, + profile_command=args.profile_command, **vars(args.command_args), ) except KeyboardInterrupt as ki: diff --git a/python/mach/mach/registrar.py b/python/mach/mach/registrar.py index e2b54b5f8cb3..be4425491c07 100644 --- a/python/mach/mach/registrar.py +++ b/python/mach/mach/registrar.py @@ -5,6 +5,8 @@ from __future__ import absolute_import, print_function, unicode_literals import time +from cProfile import Profile +from pathlib import Path import six @@ -85,7 +87,9 @@ class MachRegistrar(object): return fail_conditions - def _run_command_handler(self, handler, context, debug_command=False, **kwargs): + def _run_command_handler( + self, handler, context, debug_command=False, profile_command=False, **kwargs + ): instance = MachRegistrar._instance(handler, context, **kwargs) fail_conditions = MachRegistrar._fail_conditions(handler, instance) if fail_conditions: @@ -97,6 +101,11 @@ class MachRegistrar(object): self.command_depth += 1 fn = handler.func + profile = None + if profile_command: + profile = Profile() + profile.enable() + start_time = time.time() if debug_command: @@ -108,6 +117,19 @@ class MachRegistrar(object): end_time = time.time() + if profile_command: + profile.disable() + profile_file = ( + Path(context.topdir) / f"mach_profile_{handler.name}.cProfile" + ) + profile.dump_stats(profile_file) + print( + f'Mach command profile created at "{profile_file}". To visualize, use ' + f"snakeviz:" + ) + print("python3 -m pip install snakeviz") + print(f"python3 -m snakeviz {profile_file.name}") + result = result or 0 assert isinstance(result, six.integer_types) diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py index 96f2b6a5d8d5..c4e4d5366897 100755 --- a/testing/mozharness/mozharness/mozilla/building/buildbase.py +++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py @@ -812,19 +812,7 @@ items from that key's value." ) def _query_mach(self): - dirs = self.query_abs_dirs() - - if "MOZILLABUILD" in os.environ: - # We found many issues with intermittent build failures when not - # invoking mach via bash. - # See bug 1364651 before considering changing. - mach = [ - os.path.join(os.environ["MOZILLABUILD"], "msys", "bin", "bash.exe"), - os.path.join(dirs["abs_src_dir"], "mach"), - ] - else: - mach = [sys.executable, "mach"] - return mach + return [sys.executable, "mach"] def _run_mach_command_in_build_env(self, args, use_subprocess=False): """Run a mach command in a build context."""