corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1730712: Command virtualenvs should include Mach's import scope. r=ahal

Browse Source 
Mach's import scope includes:
* Its `pth` entries
* Its pip packages, which is either:
    * The Mach virtualenv site-packages, if packages were "pip
      installed" over the internet.
    * The system environment's site-packages, if installing packages
      over the internet is disabled and Mach is grabbing packages from
      the system Python instead.

Command virtualenvs _already_ had this import scope when they were
dynamically activated from an existing Mach process. However, when
used directly (e.g. by `<venv>/bin/python <script>`), they would be
missing this import scope, which was a confusing inconsistency.

However, resolving this inconsistency adds a new risk: when Mach is
using the system Python, the system packages now populate the same
context as command virtualenv packages - and they hadn't been checked
for compatibility. So, this patch also
includes behaviour to verify system<=>command-venv compatibility at
activation-time.

A few notes about this system-package-verification:
* It happens at virtualenv activation-time instead of build-time because
  system packages may change after the virtualenv is built.
* It takes roughly 1.5s, which is significant, but it should mainly occur
  in CI, where it's acceptable. Devs using `MACH_USE_SYSTEM_PACKAGES`
  should unset the variable to avoid the time delay.
* The algorithm works by asserting top-level requirements (e.g.
  `psutil>=5.4.2,<=5.8.0`), then runs `pip check` over the combined set
  of packages that would be in-scope.
* Note that, in this patch, system packages are *not* asserted to be
  the same version as their vendored counterparts. This is because, until
  we parse `third_party/python/requirements.in`, we don't know which
  packages we depend on directly (and whose APIs we care about if they
  change). Since leaning on system packages is essentially only used in
  CI (which we have strong control on), this downside seemed acceptable.

Differential Revision: https://phabricator.services.mozilla.com/D126288
This commit is contained in:
Mitchell Hentges
2021-11-24 20:06:32 +00:00
parent 2e259c38a5
commit d5219f4f00
8 changed files with 552 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
configure.py
View File

@@ -22,7 +22,8 @@ sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing"))
sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
from mach.site import CommandSiteManager
from mach.site import CommandSiteManager, MachSiteManager, MozSiteMetadata
from mozboot.util import get_state_dir
from mozbuild.configure import (
ConfigureSandbox,
TRACE,
@@ -226,16 +227,38 @@ def config_status(config, execute=True):
def _activate_build_virtualenv():
"""Ensure that the build virtualenv is activated
configure.py may be executed through Mach, or via "./configure, make".
In the first case, the build virtualenv should already be activated.
In the second case, we're likely being executed with the system Python, and must
prepare the virtualenv and activate it ourselves.
"""
version = ".".join(str(i) for i in sys.version_info[0:3])
print(f"Using Python {version} from {sys.executable}")
active_site = MozSiteMetadata.from_runtime()
if active_site and active_site.site_name == "build":
# We're already running within the "build" virtualenv, no additional work is
# needed.
return
# We're using the system python (or are nested within a non-build mach-managed
# virtualenv), so we should activate the build virtualenv as expected by the rest of
# configure.
topobjdir = os.path.realpath(".")
topsrcdir = os.path.realpath(os.path.dirname(__file__))
state_dir = get_state_dir()
build_site = CommandSiteManager(
mach_site = MachSiteManager.from_environment(topsrcdir, state_dir)
mach_site.activate()
build_site = CommandSiteManager.from_environment(
topsrcdir,
os.path.join(topobjdir, "_virtualenvs"),
state_dir,
"build",
os.path.join(topobjdir, "_virtualenvs"),
)
if not build_site.ensure():
print("Created Python 3 virtualenv")

574
python/mach/mach/site.py
View File

@@ -18,8 +18,12 @@ import site
import subprocess
import sys
from pathlib import Path
import tempfile
from typing import Optional
from mach.requirements import MachEnvRequirements
from mach.requirements import (
MachEnvRequirements,
)
PTH_FILENAME = "mach.pth"
METADATA_FILENAME = "moz_virtualenv_metadata.json"
@@ -33,18 +37,65 @@ class MozSiteMetadataOutOfDateError(Exception):
pass
class MozSiteMetadata:
"""Details about a Moz-managed python environment"""
class SitePackagesSource(enum.Enum):
NONE = enum.auto()
SYSTEM = enum.auto()
VENV = enum.auto()
class MozSiteMetadata:
"""Details about a Moz-managed python site
When a Moz-managed site is active, its associated metadata is available
at "MozSiteMetadata.current".
Sites that have associated virtualenvs (so, those that aren't strictly leaning on
the external python packages) will have their metadata written to
<prefix>/moz_virtualenv_metadata.json.
"""
# Used to track which which virtualenv has been activated in-process.
current: Optional["MozSiteMetadata"] = None
def __init__(
self,
hex_version: int,
site_name: str,
site_packages_source: SitePackagesSource,
mach_site_packages_source: SitePackagesSource,
external_python: "ExternalPythonSite",
prefix: str,
):
"""
Args:
hex_version: The python version number from sys.hexversion
site_name: The name of the site this metadata is associated with
site_packages_source: Where this site imports its
pip-installed dependencies from
mach_site_packages_source: Where the Mach site imports
its pip-installed dependencies from
external_python: The external Python site that was
used to invoke Mach. Usually the system Python, such as /usr/bin/python3
prefix: The same value as "sys.prefix" is when running within the
associated Python site. The same thing as the "virtualenv root".
"""
def __init__(self, hex_version, site_name, prefix):
self.hex_version = hex_version
self.site_name = site_name
self.site_packages_source = site_packages_source
self.mach_site_packages_source = mach_site_packages_source
# external_python_executable is needed for commands that tweak the system, such
# as "./mach install-moz-phab".
self.external_python = external_python
self.prefix = prefix
def write(self, is_finalized):
raw = {
"hex_version": self.hex_version,
"virtualenv_name": self.site_name,
"site_packages_source": self.site_packages_source.name,
"mach_site_packages_source": self.mach_site_packages_source.name,
"external_python_executable": self.external_python.python_path,
"is_finalized": is_finalized,
}
with open(os.path.join(self.prefix, METADATA_FILENAME), "w") as file:
@@ -55,10 +106,16 @@ class MozSiteMetadata:
type(self) == type(other)
and self.hex_version == other.hex_version
and self.site_name == other.site_name
and self.site_packages_source == other.site_packages_source
and self.mach_site_packages_source == other.mach_site_packages_source
and self.external_python.python_path == other.external_python.python_path
)
@classmethod
def from_runtime(cls):
if cls.current:
return cls.current
return cls.from_path(sys.prefix)
@classmethod
@@ -77,7 +134,10 @@ class MozSiteMetadata:
return cls(
raw["hex_version"],
raw["virtualenv_name"],
prefix,
SitePackagesSource[raw["site_packages_source"]],
SitePackagesSource[raw["mach_site_packages_source"]],
ExternalPythonSite(raw["external_python_executable"]),
metadata_path,
)
except FileNotFoundError:
return None
@@ -85,12 +145,6 @@ class MozSiteMetadata:
raise out_of_date_exception
class SitePackagesSource(enum.Enum):
NONE = enum.auto()
SYSTEM = enum.auto()
VENV = enum.auto()
class MachSiteManager:
"""Represents the activate-able "import scope" Mach needs
@@ -105,27 +159,45 @@ class MachSiteManager:
def __init__(
self,
topsrcdir,
state_dir,
requirements,
active_metadata,
site_packages_source,
topsrcdir: str,
state_dir: str,
requirements: MachEnvRequirements,
external_python: "ExternalPythonSite",
site_packages_source: SitePackagesSource,
):
"""
Args:
topsrcdir: The path to the Firefox repo
state_dir: The path to the state_dir, generally ~/.mozbuild
requirements: The requirements associated with the Mach site, parsed from
the file at build/mach_virtualenv_packages.txt
external_python: The external Python site that was used to invoke Mach.
Usually the system Python, such as /usr/bin/python3
site_packages_source: Where the Mach site will import its pip-installed
dependencies from
"""
self._topsrcdir = topsrcdir
self._active_metadata = active_metadata
self._external_python = external_python
self._site_packages_source = site_packages_source
self._requirements = requirements
self._virtualenv_root = os.path.join(state_dir, "_virtualenvs", "mach")
self._metadata = MozSiteMetadata(
sys.hexversion,
"mach",
self._virtualenv_root,
site_packages_source,
site_packages_source,
external_python,
_mach_virtualenv_root(state_dir),
)
@classmethod
def from_environment(
cls, topsrcdir, state_dir, is_mach_create_mach_env_command=False
):
def from_environment(cls, topsrcdir: str, state_dir: str):
"""
Args:
topsrcdir: The path to the Firefox repo
state_dir: The path to the state_dir, generally ~/.mozbuild
"""
requirements = _requirements(topsrcdir, "mach")
# Mach needs to operate in environments in which no pip packages are installed
# yet, and the system isn't guaranteed to have the packages we need. For example,
@@ -135,49 +207,42 @@ class MachSiteManager:
not requirements.pypi_requirements
), "Mach pip package requirements must be optional."
if not (
os.environ.get("MACH_USE_SYSTEM_PYTHON") or os.environ.get("MOZ_AUTOMATION")
):
active_metadata = MozSiteMetadata.from_runtime()
if active_metadata and active_metadata.site_name == "mach":
source = SitePackagesSource.VENV
elif not is_mach_create_mach_env_command:
# We're in an environment where we normally *would* use the Mach
# virtualenv, but we're running a "nativecmd" such as
# "create-mach-environment". So, operate Mach without leaning on the
# on-disk virtualenv or system packages.
source = SitePackagesSource.NONE
else:
# This catches the case where "./mach create-mach-environment" was run:
# * On Mach initialization, the previous clause is hit, and Mach
# operates without any extra site packages
# * When the actual "create-mach-environment" function runs,
# MachPythonEnvironment.from_environment() runs again, and this clause
# is triggered. We set the site packages source to the on-disk venv so
# it gets populated.
source = SitePackagesSource.VENV
active_metadata = MozSiteMetadata.from_runtime()
if active_metadata:
external_python = active_metadata.external_python
else:
external_python = ExternalPythonSite(sys.executable)
if external_python.has_pip():
if os.environ.get("MACH_USE_SYSTEM_PYTHON") and not os.environ.get(
"MOZ_AUTOMATION"
):
print(
"Since Mach has been requested to use the system Python "
"environment, it will need to verify compatibility before "
"running the current command. This may take a couple seconds.\n"
"Note: you can avoid this delay by unsetting the "
"MACH_USE_SYSTEM_PYTHON environment variable."
)
source = SitePackagesSource.SYSTEM
else:
source = SitePackagesSource.NONE
if not _system_python_env_variable_present():
# We can detect that this is for initializing a native command by:
# * Checking if we're not using the Mach virtualenv's python, yet the
# "use system python" environment variable isn't set.
# * We're still in the "initialization" phase because there's no "current"
# MozEnvironment active yet (so, when the `./mach create-mach-environment`
# command's function runs, this check will be Falsy).
# If true, then don't use any "site-packages" sources.
is_for_initializing_native_cmd = (
not active_metadata or active_metadata.site_name != "mach"
) and not MozSiteMetadata.current
source = (
SitePackagesSource.NONE
if is_for_initializing_native_cmd
else SitePackagesSource.VENV
)
elif not external_python.has_pip():
source = SitePackagesSource.NONE
else:
source = (
SitePackagesSource.SYSTEM
if external_python.provides_any_package("mach", requirements)
else SitePackagesSource.NONE
)
return cls(
topsrcdir,
state_dir,
requirements,
MozSiteMetadata.from_runtime(),
external_python,
source,
)
@@ -185,15 +250,18 @@ class MachSiteManager:
if self._site_packages_source == SitePackagesSource.NONE:
return True
elif self._site_packages_source == SitePackagesSource.SYSTEM:
_assert_pip_check()
external_python = ExternalPythonSite(sys.executable)
external_python.validate_mach_packages(self._requirements)
pthfile_lines = [
*self._requirements.pths_as_absolute(self._topsrcdir),
*self._external_python.all_site_packages_dirs(),
]
_assert_pip_check(self._topsrcdir, pthfile_lines, "mach")
return True
elif self._site_packages_source == SitePackagesSource.VENV:
environment = self._virtualenv()
return _is_venv_up_to_date(
self._topsrcdir,
environment,
self._pthfile_lines(environment),
self._requirements,
self._metadata,
)
@@ -212,6 +280,8 @@ class MachSiteManager:
return up_to_date
def activate(self):
assert not MozSiteMetadata.current
self.ensure()
if self._site_packages_source == SitePackagesSource.SYSTEM:
# Add our Mach modules to the sys.path.
@@ -248,6 +318,8 @@ class MachSiteManager:
activate_path = self._virtualenv().activate_path
exec(open(activate_path).read(), dict(__file__=activate_path))
MozSiteMetadata.current = self._metadata
def _build(self):
if self._site_packages_source != SitePackagesSource.VENV:
# The Mach virtualenv doesn't have a physical virtualenv on-disk if it won't
@@ -258,51 +330,149 @@ class MachSiteManager:
_create_venv_with_pthfile(
self._topsrcdir,
environment,
self._pthfile_lines(environment),
self._site_packages_source,
self._requirements,
self._metadata,
)
def _pthfile_lines(self, environment):
return [
# Prioritize vendored and first-party modules first.
*self._requirements.pths_as_absolute(self._topsrcdir),
# Then, include the virtualenv's site-packages.
*_deprioritize_venv_packages(environment.site_packages_dir()),
]
def _virtualenv(self):
assert self._site_packages_source == SitePackagesSource.VENV
return PythonVirtualenv(self._metadata.prefix)
class CommandSiteManager:
"""Public interface for activating virtualenvs and ad-hoc-installing pip packages"""
"""Activate sites and ad-hoc-install pip packages
Provides tools to ensure that a command's scope will have expected, compatible
packages. Manages prioritization of the import scope, and ensures consistency
regardless of how a virtualenv is used (whether via in-process activation, or when
used standalone to invoke a script).
A few notes:
* The command environment always inherits Mach's import scope. This is because
"unloading" packages in Python is error-prone, so in-process activations will always
carry Mach's dependencies along with it. Accordingly, compatibility between each
command environment and the Mach environment must be maintained
* Unlike the Mach environment, command environments *always* have an associated
physical virtualenv on-disk. This is because some commands invoke child Python
processes, and that child process should have the same import scope.
"""
def __init__(
self,
topsrcdir,
virtualenvs_dir,
site_name,
topsrcdir: str,
state_dir: str,
virtualenv_root: str,
site_name: str,
active_metadata: MozSiteMetadata,
site_packages_source: SitePackagesSource,
requirements: MachEnvRequirements,
):
self.topsrcdir = topsrcdir
self._virtualenv_name = site_name
self.virtualenv_root = os.path.join(virtualenvs_dir, site_name)
"""
Args:
topsrcdir: The path to the Firefox repo
state_dir: The path to the state_dir, generally ~/.mozbuild
virtualenv_root: The path to the virtualenv associated with this site
site_name: The name of this site, such as "build"
active_metadata: The currently-active moz-managed site
site_packages_source: Where this site will import its pip-installed
dependencies from
requirements: The requirements associated with this site, parsed from
the file at build/<site_name>_virtualenv_packages.txt
"""
self._topsrcdir = topsrcdir
self._state_dir = state_dir
self.virtualenv_root = virtualenv_root
self._site_name = site_name
self._virtualenv = PythonVirtualenv(self.virtualenv_root)
self.bin_path = self._virtualenv.bin_path
self.python_path = self._virtualenv.python_path
self.bin_path = self._virtualenv.bin_path
self._site_packages_source = site_packages_source
self._mach_site_packages_source = active_metadata.mach_site_packages_source
self._external_python = active_metadata.external_python
self._requirements = requirements
self._metadata = MozSiteMetadata(
sys.hexversion,
site_name,
self._virtualenv.prefix,
site_packages_source,
active_metadata.mach_site_packages_source,
active_metadata.external_python,
virtualenv_root,
)
@classmethod
def from_environment(
cls,
topsrcdir: str,
state_dir: str,
site_name: str,
command_virtualenvs_dir: str,
):
"""
Args:
topsrcdir: The path to the Firefox repo
state_dir: The path to the state_dir, generally ~/.mozbuild
site_name: The name of this site, such as "build"
command_virtualenvs_dir: The location under which this site's virtualenv
should be created
"""
active_metadata = MozSiteMetadata.from_runtime()
assert (
active_metadata
), "A Mach-managed site must be active before doing work with command sites"
requirements = _requirements(topsrcdir, site_name)
if not _system_python_env_variable_present() or site_name != "build":
source = SitePackagesSource.VENV
elif not active_metadata.external_python.has_pip():
if requirements.pypi_requirements:
raise Exception(
f'The "{site_name}" site requires pip '
"packages, and Mach has been told to find such pip packages in "
"the system environment, but it can't because the system doesn't "
'have "pip" installed.'
)
source = SitePackagesSource.NONE
else:
source = (
SitePackagesSource.SYSTEM
if active_metadata.external_python.provides_any_package(
site_name, requirements
)
else SitePackagesSource.NONE
)
return cls(
topsrcdir,
state_dir,
os.path.join(command_virtualenvs_dir, site_name),
site_name,
active_metadata,
source,
requirements,
)
def ensure(self):
"""Ensure the site is present and up to date."""
requirements = _requirements(self.topsrcdir, self._virtualenv_name)
if not _is_venv_up_to_date(
self.topsrcdir,
self._virtualenv,
requirements,
self._metadata,
):
"""Ensure that this virtualenv is built, up-to-date, and ready for use
If using a virtualenv Python binary directly, it's useful to call this function
first to ensure that the virtualenv doesn't have obsolete references or packages.
"""
if not self._up_to_date():
_create_venv_with_pthfile(
self.topsrcdir,
self._topsrcdir,
self._virtualenv,
SitePackagesSource.VENV,
requirements,
self._pthfile_lines(),
self._site_packages_source,
self._requirements,
self._metadata,
)
@@ -317,6 +487,7 @@ class CommandSiteManager:
self.ensure()
activate_path = self._virtualenv.activate_path
exec(open(activate_path).read(), dict(__file__=activate_path))
MozSiteMetadata.current = self._metadata
def install_pip_package(self, package):
"""Install a package via pip.
@@ -326,7 +497,7 @@ class CommandSiteManager:
If the package is already installed, this is a no-op.
"""
if os.path.normcase(sys.executable).startswith(os.path.normcase(self.bin_path)):
if Path(sys.prefix) == Path(self.virtualenv_root):
# If we're already running in this interpreter, we can optimize in
# the case that the package requirement is already satisfied.
from pip._internal.req.constructors import install_req_from_line
@@ -350,7 +521,7 @@ class CommandSiteManager:
"""
if not os.path.isabs(path):
path = os.path.join(self.topsrcdir, path)
path = os.path.join(self._topsrcdir, path)
args = ["--requirement", path]
@@ -362,6 +533,83 @@ class CommandSiteManager:
self._virtualenv.pip_install(args)
def _pthfile_lines(self):
"""Generate the prioritized import scope to encode in the venv's pthfile
The import priority looks like this:
1. Mach's vendored/first-party modules
2. Mach's site-package source (the Mach virtualenv, the system Python, or neither)
3. The command's vendored/first-party modules
4. The command's site-package source (either the virtualenv or the system Python,
if it's not already added)
Note that, when using the system Python, it may either be prioritized before or
after the command's vendored/first-party modules. This is a symptom of us
attempting to avoid conflicting with the system packages.
For example, there's at least one job in CI that operates with an ancient
environment with a bunch of old packages, many of whom conflict with our vendored
packages. However, the specific command that we're running for the job doesn't
need any of the system's packages, so we're safe to insulate ourselves.
Mach doesn't know the command being run when it's preparing its import scope,
so it has to be defensive. Therefore:
1. If Mach needs a system package: system packages are higher priority.
2. If Mach doesn't need a system package, but the current command does: system
packages are still be in the list, albeit at a lower priority.
"""
# Prioritize Mach's vendored and first-party modules first.
lines = _requirements(self._topsrcdir, "mach").pths_as_absolute(self._topsrcdir)
mach_site_packages_source = self._mach_site_packages_source
if mach_site_packages_source == SitePackagesSource.SYSTEM:
# When Mach is using the system environment, add it next.
lines.extend(self._external_python.all_site_packages_dirs())
elif mach_site_packages_source == SitePackagesSource.VENV:
# When Mach is using its on-disk virtualenv, add its site-packages directory.
lines.append(
PythonVirtualenv(
_mach_virtualenv_root(self._state_dir)
).site_packages_dir()
)
# Add this command's vendored and first-party modules.
lines.extend(self._requirements.pths_as_absolute(self._topsrcdir))
if (
self._site_packages_source == SitePackagesSource.SYSTEM
and not mach_site_packages_source == SitePackagesSource.SYSTEM
):
# Only add the system python site-packages if it wasn't already added
# earlier for Mach.
lines.extend(self._external_python.all_site_packages_dirs())
elif self._site_packages_source == SitePackagesSource.VENV:
# When allowed to pip install to the on-disk virtualenv, ensure that its
# site-packages is in-scope at the end of the list.
site_packages_dir = self._virtualenv.site_packages_dir()
lines.extend(_deprioritize_venv_packages(site_packages_dir))
# Note that an on-disk virtualenv is always created for commands, even if they
# are using the system as their site-packages source. Also note that, in such
# a case, we aren't "deprioritizing" the venv_packages. This is because:
# * There should be no risk of breakage, since *nothing* should be "pip install"-d
# into the virtualenv, and
# * There's nontrivial complexity in purging the virtualenv's site-packages from
# the sys.path after activation, but since it isn't doing any harm in staying,
# we leave it.
return lines
def _up_to_date(self):
if self._site_packages_source == SitePackagesSource.SYSTEM:
_assert_pip_check(self._topsrcdir, self._pthfile_lines(), self._site_name)
return _is_venv_up_to_date(
self._topsrcdir,
self._virtualenv,
self._pthfile_lines(),
self._requirements,
self._metadata,
)
class PythonVirtualenv:
"""Calculates paths of interest for general python virtual environments"""
@@ -433,6 +681,7 @@ class ExternalSitePackageValidationResult:
def __init__(self):
self._package_discrepancies = []
self.has_all_packages = True
self.provides_any_package = False
def add_discrepancy(self, requirement, found):
self._package_discrepancies.append((requirement, found))
@@ -475,6 +724,9 @@ class ExternalPythonSite:
"import site; print([site.getusersitepackages()] "
"+ site.getsitepackages())",
],
# The "site" module may return erroneous entries for the system python
# if the "VIRTUAL_ENV" environment variable is set.
env={k: v for k, v in os.environ.items() if k != "VIRTUAL_ENV"},
universal_newlines=True,
)
return ast.literal_eval(paths_string)
@@ -488,27 +740,48 @@ class ExternalPythonSite:
== 0
)
def validate_mach_packages(self, requirements):
def provides_any_package(self, virtualenv_name, requirements):
system_packages = self._resolve_installed_packages()
result = ExternalSitePackageValidationResult()
for pkg in requirements.pypi_requirements:
installed_version = system_packages.get(pkg.requirement.name)
if not installed_version or not pkg.requirement.specifier.contains(
installed_version
):
result.add_discrepancy(pkg.requirement, installed_version)
elif installed_version:
result.provides_any_package = True
for pkg in requirements.pypi_optional_requirements:
installed_version = system_packages.get(pkg.requirement.name)
if installed_version and not pkg.requirement.specifier.contains(
installed_version
):
result.add_discrepancy(pkg.requirement, installed_version)
elif installed_version:
result.provides_any_package = True
if not result.has_all_packages:
print(result.report())
raise Exception(
"Your system Python packages aren't compatible with the "
'"Mach" virtualenv'
f'The Python packages associated with "{self.python_path}" aren\'t '
f'compatible with the "{virtualenv_name}" virtualenv'
)
return result.provides_any_package
@functools.lru_cache(maxsize=None)
def _resolve_installed_packages(self):
pip_json = subprocess.check_output(
[self.python_path, "-m", "pip", "list", "--format", "json"],
[
self.python_path,
"-m",
"pip",
"list",
"--format",
"json",
"--disable-pip-version-check",
],
universal_newlines=True,
)
@@ -540,24 +813,114 @@ def _requirements(topsrcdir, virtualenv_name):
)
def _assert_pip_check():
"""Check that the current site's packages are compatible with each other"""
pip = [sys.executable, "-m", "pip"]
check_result = subprocess.run(
pip + ["check"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
def _virtualenv_py_path(topsrcdir):
return os.path.join(
topsrcdir, "third_party", "python", "virtualenv", "virtualenv.py"
)
if check_result.returncode:
print(check_result.stdout, file=sys.stderr)
subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr)
raise Exception(
'According to "pip check", the current Python site '
f'(associated with "{sys.executable}") has package-compatibility issues.'
def _system_python_env_variable_present():
return any(
os.environ.get(var) for var in ("MACH_USE_SYSTEM_PYTHON", "MOZ_AUTOMATION")
)
def _assert_pip_check(topsrcdir, pthfile_lines, virtualenv_name):
"""Check if the provided pthfile lines have a package incompatibility
If there's an incompatibility, raise an exception and allow it to bubble up since
it will require user intervention to resolve.
"""
if os.environ.get(
f"MACH_SYSTEM_ASSERTED_COMPATIBLE_WITH_{virtualenv_name.upper()}_SITE", None
):
# Don't re-assert compatibility against the system python within Mach subshells.
return
if (
virtualenv_name == "mach"
and os.environ.get("MACH_USE_SYSTEM_PYTHON")
and not os.environ.get("MOZ_AUTOMATION")
):
# Since this assertion takes some time, warn users who have explicitly opted
# in. Since we only want this message to be printed once, only do it for the
# first virtualenv that's used (which is always "mach").
print(
"Since Mach has been requested to use the system Python "
"environment, it will need to verify compatibility before "
"running the current command. This may take a couple seconds.\n"
"Note: you can avoid this delay by unsetting the "
"MACH_USE_SYSTEM_PYTHON environment variable."
)
with tempfile.TemporaryDirectory() as check_env_path:
# Pip detects packages on the "sys.path" that have a ".dist-info" or
# a ".egg-info" directory. The majority of our Python dependencies are
# vendored as extracted wheels or sdists, so they are automatically picked up.
# This gives us sufficient confidence to do a `pip check` with both vendored
# packages + system packages in scope, and trust the results.
# Note: rather than just running the system pip with a modified "sys.path",
# we create a new virtualenv that has our pinned pip version, so that
# we get consistent results (there's been lots of pip resolver behaviour
# changes recently).
subprocess.check_call(
[
sys.executable,
_virtualenv_py_path(topsrcdir),
"--no-download",
check_env_path,
],
stdout=subprocess.DEVNULL,
)
check_env = PythonVirtualenv(check_env_path)
with open(
os.path.join(os.path.join(check_env.site_packages_dir()), PTH_FILENAME),
"w",
) as f:
f.write("\n".join(pthfile_lines))
pip = [check_env.python_path, "-m", "pip"]
check_result = subprocess.run(
pip + ["check"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
)
if check_result.returncode:
print(check_result.stdout, file=sys.stderr)
subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr)
raise Exception(
'According to "pip check", the current Python '
"environment has package-compatibility issues."
)
os.environ[
f"MACH_SYSTEM_ASSERTED_COMPATIBLE_WITH_{virtualenv_name.upper()}_SITE"
] = "1"
def _deprioritize_venv_packages(site_packages_dir):
# Move the virtualenv's site-packages to the bottom so that vendored packages
# are prioritized.
# repr(...) is needed to ensure Windows path backslashes aren't mistaken for
# escape sequences.
# Additionally, when removing the existing "site-packages" entry, we have to
# do it in a case-insensitive way because, on Windows:
# * Python adds it as <venv>/lib/site-packages
# * While distutils tells us it's <venv>/Lib/site-packages
# * (note: on-disk, it's capitalized, so distutils is slightly more accurate).
return (
"import sys; sys.path = [p for p in sys.path if "
f"p.lower() != {repr(site_packages_dir)}.lower()]",
f"import sys; sys.path.append({repr(site_packages_dir)})",
)
def _create_venv_with_pthfile(
topsrcdir,
target_venv,
pthfile_lines,
site_packages_source,
requirements,
metadata,
@@ -572,13 +935,7 @@ def _create_venv_with_pthfile(
subprocess.check_call(
[
sys.executable,
os.path.join(
topsrcdir,
"third_party",
"python",
"virtualenv",
"virtualenv.py",
),
_virtualenv_py_path(topsrcdir),
# pip, setuptools and wheel are vendored and inserted into the virtualenv
# scope automatically, so "virtualenv" doesn't need to seed it.
"--no-seed",
@@ -589,7 +946,7 @@ def _create_venv_with_pthfile(
os.utime(target_venv.activate_path, None)
site_packages_dir = target_venv.site_packages_dir()
pthfile_contents = "\n".join(requirements.pths_as_absolute(topsrcdir))
pthfile_contents = "\n".join(pthfile_lines)
with open(os.path.join(site_packages_dir, PTH_FILENAME), "w") as f:
f.write(pthfile_contents)
@@ -613,6 +970,7 @@ def _create_venv_with_pthfile(
def _is_venv_up_to_date(
topsrcdir,
target_venv,
expected_pthfile_lines,
requirements,
expected_metadata,
):
@@ -658,8 +1016,12 @@ def _is_venv_up_to_date(
except FileNotFoundError:
return False
expected_pthfile_contents = "\n".join(requirements.pths_as_absolute(topsrcdir))
expected_pthfile_contents = "\n".join(expected_pthfile_lines)
if current_pthfile_contents != expected_pthfile_contents:
return False
return True
def _mach_virtualenv_root(state_dir):
return os.path.join(state_dir, "_virtualenvs", "mach")

27
python/mach/mach/test/test_decorators.py
View File

@@ -5,10 +5,13 @@
from __future__ import absolute_import, unicode_literals
import os
from unittest import mock
import pytest
from unittest.mock import Mock
from mach.requirements import MachEnvRequirements
from mach.site import CommandSiteManager, SitePackagesSource, MozSiteMetadata
from mozunit import main
import mach.registrar
@@ -91,10 +94,26 @@ def test_register_command_sets_up_class_at_runtime(registrar):
)
inner_function("bar")
registrar.dispatch("cmd_foo", context)
inner_function.assert_called_with("foo")
registrar.dispatch("cmd_bar", context)
inner_function.assert_called_with("bar")
def from_environment_patch(topsrcdir, state_dir, virtualenv_name, dir):
return CommandSiteManager(
"",
"",
virtualenv_name,
virtualenv_name,
MozSiteMetadata(
0, "mach", SitePackagesSource.VENV, SitePackagesSource.VENV, "", ""
),
SitePackagesSource.VENV,
MachEnvRequirements(),
)
with mock.patch.object(
CommandSiteManager, "from_environment", from_environment_patch
):
registrar.dispatch("cmd_foo", context)
inner_function.assert_called_with("foo")
registrar.dispatch("cmd_bar", context)
inner_function.assert_called_with("bar")
def test_cannot_create_command_nonexisting_category(registrar):

6
python/mozbuild/mozbuild/base.py
View File

@@ -290,12 +290,14 @@ class MozbuildObject(ProcessExecutionMixin):
@property
def virtualenv_manager(self):
from mach.site import CommandSiteManager
from mozboot.util import get_state_dir
if self._virtualenv_manager is None:
self._virtualenv_manager = CommandSiteManager(
self._virtualenv_manager = CommandSiteManager.from_environment(
self.topsrcdir,
os.path.join(self.topobjdir, "_virtualenvs"),
get_state_dir(),
self._virtualenv_name,
os.path.join(self.topobjdir, "_virtualenvs"),
)
return self._virtualenv_manager

19
python/mozbuild/mozbuild/controller/building.py
View File

@@ -23,13 +23,14 @@ from collections import (
)
from textwrap import TextWrapper
from mach.site import CommandSiteManager
try:
import psutil
except Exception:
psutil = None
from mach.mixin.logging import LoggingMixin
from mozboot.util import get_mach_virtualenv_binary
import mozfile
from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
from mozterm.widgets import Footer
@@ -1508,15 +1509,15 @@ class BuildDriver(MozbuildObject):
if eq == "=":
append_env[k] = v
if six.PY3:
python = sys.executable
else:
# Try to get the mach virtualenv Python if we can.
python = get_mach_virtualenv_binary()
if not os.path.exists(python):
python = "python3"
build_site = CommandSiteManager.from_environment(
self.topsrcdir,
self.statedir,
"build",
os.path.join(self.topobjdir, "_virtualenvs"),
)
build_site.ensure()
command = [python, os.path.join(self.topsrcdir, "configure.py")]
command = [build_site.python_path, os.path.join(self.topsrcdir, "configure.py")]
if options:
command.extend(options)

1
python/mozbuild/mozbuild/mach_commands.py
View File

@@ -2463,7 +2463,6 @@ def create_mach_environment(command_context, force=False):
manager = MachSiteManager.from_environment(
command_context.topsrcdir,
get_state_dir(),
is_mach_create_mach_env_command=True,
)
if manager.ensure(force=force):

6
python/mozbuild/mozbuild/sphinx.py
View File

@@ -198,6 +198,7 @@ class MozbuildSymbols(Directive):
def setup(app):
from mach.site import CommandSiteManager
from mozboot.util import get_state_dir
from moztreedocs import manager
app.add_directive("mozbuildsymbols", MozbuildSymbols)
@@ -214,9 +215,10 @@ def setup(app):
# We need to adjust sys.path in order for Python API docs to get generated
# properly. We leverage the in-tree virtualenv for this.
topsrcdir = manager.topsrcdir
site = CommandSiteManager(
site = CommandSiteManager.from_environment(
topsrcdir,
os.path.join(app.outdir, "_venv"),
get_state_dir(),
"common",
os.path.join(app.outdir, "_venv"),
)
site.activate()

30
python/mozbuild/mozbuild/test/test_vendor.py
View File

@@ -17,6 +17,17 @@ import mozunit
def test_up_to_date_vendor():
with tempfile.TemporaryDirectory() as work_dir:
def copy_to_work_dir(relative_path):
args = (
os.path.join(topsrcdir, relative_path),
os.path.join(work_dir, relative_path),
)
shutil.copytree(*args) if os.path.isdir(relative_path) else shutil.copy(
*args
)
subprocess.check_call(["hg", "init", work_dir])
os.makedirs(os.path.join(work_dir, "build"))
os.makedirs(os.path.join(work_dir, "third_party"))
@@ -34,17 +45,10 @@ def test_up_to_date_vendor():
file.write("vendored:third_party/python/setuptools\n")
file.write("vendored:third_party/python/wheel\n")
# Copy existing "third_party/python/" vendored files
existing_vendored = os.path.join(topsrcdir, "third_party", "python")
work_vendored = os.path.join(work_dir, "third_party", "python")
shutil.copytree(existing_vendored, work_vendored)
# Copy "mach" module so that `CommandSiteManager` can populate itself.
# This is needed because "topsrcdir" is used in this test both for determining
# import paths and for acting as a "work dir".
existing_mach = os.path.join(topsrcdir, "python", "mach")
work_mach = os.path.join(work_dir, "python", "mach")
shutil.copytree(existing_mach, work_mach)
copy_to_work_dir(os.path.join("third_party", "python"))
copy_to_work_dir(os.path.join("python", "mach"))
copy_to_work_dir(os.path.join("build", "mach_virtualenv_packages.txt"))
copy_to_work_dir(os.path.join("build", "common_virtualenv_packages.txt"))
# Run the vendoring process
vendor = VendorPython(
@@ -59,8 +63,8 @@ def test_up_to_date_vendor():
[
"diff",
"-r",
existing_vendored,
work_vendored,
os.path.join(topsrcdir, os.path.join("third_party", "python")),
os.path.join(work_dir, os.path.join("third_party", "python")),
"--exclude=__pycache__",
]
)