corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1696251 - Replace self with command_context where possible in existing mach commands. r=mhentges,webdriver-reviewers,perftest-reviewers,whimboo

Browse Source 
This step removes all the dependencies of mach commands to
having a MachCommandBase as the `self` by using the `command_context`
argument instead. This also removes any remaining statefulness from those
classes that implement mach commands, ultimately making it easier to move
existing commands out of classes in a follow-up.

Differential Revision: https://phabricator.services.mozilla.com/D118058
This commit is contained in:
Alex Lopez
2021-07-16 15:51:29 +00:00
parent e94ed65815
commit eaf2553782
53 changed files with 1813 additions and 1425 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
python/mach_commands.py
View File

@@ -63,7 +63,7 @@ class MachCommands(MachCommandBase):
args,
):
# Avoid logging the command
self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
command_context.log_manager.terminal_handler.setLevel(logging.CRITICAL)
# Note: subprocess requires native strings in os.environ on Windows.
append_env = {"PYTHONDONTWRITEBYTECODE": str("1")}
@@ -75,14 +75,16 @@ class MachCommands(MachCommandBase):
from mach_bootstrap import mach_sys_path
python_path = sys.executable
append_env["PYTHONPATH"] = os.pathsep.join(mach_sys_path(self.topsrcdir))
append_env["PYTHONPATH"] = os.pathsep.join(
mach_sys_path(command_context.topsrcdir)
)
else:
self.virtualenv_manager.ensure()
command_context.virtualenv_manager.ensure()
if not no_activate:
self.virtualenv_manager.activate()
python_path = self.virtualenv_manager.python_path
command_context.virtualenv_manager.activate()
python_path = command_context.virtualenv_manager.python_path
if requirements:
self.virtualenv_manager.install_pip_requirements(
command_context.virtualenv_manager.install_pip_requirements(
requirements, require_hashes=False
)
@@ -97,14 +99,14 @@ class MachCommands(MachCommandBase):
if not no_virtualenv:
# Use `_run_pip` directly rather than `install_pip_package` to bypass
# `req.check_if_exists()` which may detect a system installed ipython.
self.virtualenv_manager._run_pip(["install", "ipython"])
command_context.virtualenv_manager._run_pip(["install", "ipython"])
python_path = which("ipython", path=bindir)
if not python_path:
print("error: could not detect or install ipython")
return 1
return self.run_process(
return command_context.run_process(
[python_path] + args,
pass_thru=True, # Allow user to run Python interactively.
ensure_exit_code=False, # Don't throw on non-zero exit code.
@@ -169,7 +171,7 @@ class MachCommands(MachCommandBase):
os.environ[b"PYTHON_TEST_TMP"] = tempdir
else:
os.environ["PYTHON_TEST_TMP"] = tempdir
return self.run_python_tests(*args, **kwargs)
return self.run_python_tests(command_context, *args, **kwargs)
finally:
import mozfile
@@ -177,6 +179,7 @@ class MachCommands(MachCommandBase):
def run_python_tests(
self,
command_context,
tests=None,
test_objects=None,
subsuite=None,
@@ -187,11 +190,11 @@ class MachCommands(MachCommandBase):
**kwargs
):
self.activate_virtualenv()
command_context.activate_virtualenv()
if test_objects is None:
from moztest.resolve import TestResolver
resolver = self._spawn(TestResolver)
resolver = command_context._spawn(TestResolver)
# If we were given test paths, try to find tests matching them.
test_objects = resolver.resolve_tests(paths=tests, flavor="python")
else:
@@ -212,7 +215,7 @@ class MachCommands(MachCommandBase):
tests = mp.active_tests(
filters=filters,
disabled=False,
python=self.virtualenv_manager.version_info()[0],
python=command_context.virtualenv_manager.version_info()[0],
**mozinfo.info
)
@@ -222,7 +225,7 @@ class MachCommands(MachCommandBase):
"TEST-UNEXPECTED-FAIL | No tests collected "
+ "{}(Not in PYTHON_UNITTEST_MANIFESTS?)".format(submsg)
)
self.log(logging.WARN, "python-test", {}, message)
command_context.log(logging.WARN, "python-test", {}, message)
return 1
parallel = []
@@ -238,7 +241,7 @@ class MachCommands(MachCommandBase):
test.get("requirements")
and test["requirements"] not in installed_requirements
):
self.virtualenv_manager.install_pip_requirements(
command_context.virtualenv_manager.install_pip_requirements(
test["requirements"], quiet=True
)
installed_requirements.add(test["requirements"])
@@ -253,9 +256,7 @@ class MachCommands(MachCommandBase):
else:
parallel.append(test)
self.jobs = jobs or cpu_count()
self.terminate = False
self.verbose = verbose
jobs = jobs or cpu_count()
return_code = 0
@@ -263,10 +264,12 @@ class MachCommands(MachCommandBase):
output, ret, test_path = result
for line in output:
self.log(logging.INFO, "python-test", {"line": line.rstrip()}, "{line}")
command_context.log(
logging.INFO, "python-test", {"line": line.rstrip()}, "{line}"
)
if ret and not return_code:
self.log(
command_context.log(
logging.ERROR,
"python-test",
{"test_path": test_path, "ret": ret},
@@ -274,9 +277,12 @@ class MachCommands(MachCommandBase):
)
return return_code or ret
with ThreadPoolExecutor(max_workers=self.jobs) as executor:
with ThreadPoolExecutor(max_workers=jobs) as executor:
futures = [
executor.submit(self._run_python_test, test) for test in parallel
executor.submit(
self._run_python_test, command_context, test, jobs, verbose
)
for test in parallel
]
try:
@@ -290,11 +296,13 @@ class MachCommands(MachCommandBase):
raise
for test in sequential:
return_code = on_test_finished(self._run_python_test(test))
return_code = on_test_finished(
self._run_python_test(command_context, test, jobs, verbose)
)
if return_code and exitfirst:
break
self.log(
command_context.log(
logging.INFO,
"python-test",
{"return_code": return_code},
@@ -302,17 +310,19 @@ class MachCommands(MachCommandBase):
)
return return_code
def _run_python_test(self, test):
def _run_python_test(self, command_context, test, jobs, verbose):
from mozprocess import ProcessHandler
output = []
def _log(line):
# Buffer messages if more than one worker to avoid interleaving
if self.jobs > 1:
if jobs > 1:
output.append(line)
else:
self.log(logging.INFO, "python-test", {"line": line.rstrip()}, "{line}")
command_context.log(
logging.INFO, "python-test", {"line": line.rstrip()}, "{line}"
)
file_displayed_test = [] # used as boolean
@@ -332,7 +342,7 @@ class MachCommands(MachCommandBase):
_log(line)
_log(test["path"])
python = self.virtualenv_manager.python_path
python = command_context.virtualenv_manager.python_path
cmd = [python, test["path"]]
env = os.environ.copy()
if six.PY2:
@@ -353,7 +363,7 @@ class MachCommands(MachCommandBase):
"call?): {}".format(test["path"])
)
if self.verbose:
if verbose:
if return_code != 0:
_log("Test failed: {}".format(test["path"]))
else: