Bug 1730712: Command virtualenvs should include Mach's import scope. r=ahal

Mach's import scope includes:
* Its `pth` entries
* Its pip packages, which is either:
    * The Mach virtualenv site-packages, if packages were "pip
      installed" over the internet.
    * The system environment's site-packages, if installing packages
      over the internet is disabled and Mach is grabbing packages from
      the system Python instead.

Command virtualenvs _already_ had this import scope when they were
dynamically activated from an existing Mach process. However, when
used directly (e.g. by `<venv>/bin/python <script>`), they would be
missing this import scope, which was a confusing inconsistency.

However, resolving this inconsistency adds a new risk: when Mach is
using the system Python, the system packages now populate the same
context as command virtualenv packages - and they hadn't been checked
for compatibility. So, this patch also
includes behaviour to verify system<=>command-venv compatibility at
activation-time.

A few notes about this system-package-verification:
* It happens at virtualenv activation-time instead of build-time because
  system packages may change after the virtualenv is built.
* It takes roughly 1.5s, which is significant, but it should mainly occur
  in CI, where it's acceptable. Devs using `MACH_USE_SYSTEM_PACKAGES`
  should unset the variable to avoid the time delay.
* The algorithm works by asserting top-level requirements (e.g.
  `psutil>=5.4.2,<=5.8.0`), then runs `pip check` over the combined set
  of packages that would be in-scope.
* Note that, in this patch, system packages are *not* asserted to be
  the same version as their vendored counterparts. This is because, until
  we parse `third_party/python/requirements.in`, we don't know which
  packages we depend on directly (and whose APIs we care about if they
  change). Since leaning on system packages is essentially only used in
  CI (which we have strong control on), this downside seemed acceptable.

Differential Revision: https://phabricator.services.mozilla.com/D126288

configure.py

@@ -22,7 +22,8 @@ sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing"))
 sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
-from mach.site import CommandSiteManager
+from mach.site import CommandSiteManager, MachSiteManager, MozSiteMetadata
+from mozboot.util import get_state_dir
 from mozbuild.configure import (
     ConfigureSandbox,
     TRACE,
@@ -226,16 +227,38 @@ def config_status(config, execute=True):
 
 
 def _activate_build_virtualenv():
+    """Ensure that the build virtualenv is activated
+
+    configure.py may be executed through Mach, or via "./configure, make".
+    In the first case, the build virtualenv should already be activated.
+    In the second case, we're likely being executed with the system Python, and must
+    prepare the virtualenv and activate it ourselves.
+    """
+
     version = ".".join(str(i) for i in sys.version_info[0:3])
     print(f"Using Python {version} from {sys.executable}")
 
+    active_site = MozSiteMetadata.from_runtime()
+    if active_site and active_site.site_name == "build":
+        # We're already running within the "build" virtualenv, no additional work is
+        # needed.
+        return
+
+    # We're using the system python (or are nested within a non-build mach-managed
+    # virtualenv), so we should activate the build virtualenv as expected by the rest of
+    # configure.
+
     topobjdir = os.path.realpath(".")
     topsrcdir = os.path.realpath(os.path.dirname(__file__))
+    state_dir = get_state_dir()
 
-    build_site = CommandSiteManager(
+    mach_site = MachSiteManager.from_environment(topsrcdir, state_dir)
+    mach_site.activate()
+    build_site = CommandSiteManager.from_environment(
         topsrcdir,
-        os.path.join(topobjdir, "_virtualenvs"),
+        state_dir,
         "build",
+        os.path.join(topobjdir, "_virtualenvs"),
     )
     if not build_site.ensure():
         print("Created Python 3 virtualenv")

python/mach/mach/site.py

@@ -18,8 +18,12 @@ import site
 import subprocess
 import sys
 from pathlib import Path
+import tempfile
+from typing import Optional
 
-from mach.requirements import MachEnvRequirements
+from mach.requirements import (
+    MachEnvRequirements,
+)
 
 PTH_FILENAME = "mach.pth"
 METADATA_FILENAME = "moz_virtualenv_metadata.json"
@@ -33,18 +37,65 @@ class MozSiteMetadataOutOfDateError(Exception):
     pass
 
 
-class MozSiteMetadata:
+class SitePackagesSource(enum.Enum):
-    """Details about a Moz-managed python environment"""
+    NONE = enum.auto()
+    SYSTEM = enum.auto()
+    VENV = enum.auto()
+
+
+class MozSiteMetadata:
+    """Details about a Moz-managed python site
+
+    When a Moz-managed site is active, its associated metadata is available
+    at "MozSiteMetadata.current".
+
+    Sites that have associated virtualenvs (so, those that aren't strictly leaning on
+    the external python packages) will have their metadata written to
+    <prefix>/moz_virtualenv_metadata.json.
+    """
+
+    # Used to track which which virtualenv has been activated in-process.
+    current: Optional["MozSiteMetadata"] = None
+
+    def __init__(
+        self,
+        hex_version: int,
+        site_name: str,
+        site_packages_source: SitePackagesSource,
+        mach_site_packages_source: SitePackagesSource,
+        external_python: "ExternalPythonSite",
+        prefix: str,
+    ):
+        """
+        Args:
+            hex_version: The python version number from sys.hexversion
+            site_name: The name of the site this metadata is associated with
+            site_packages_source: Where this site imports its
+                pip-installed dependencies from
+            mach_site_packages_source: Where the Mach site imports
+                its pip-installed dependencies from
+            external_python: The external Python site that was
+                used to invoke Mach. Usually the system Python, such as /usr/bin/python3
+            prefix: The same value as "sys.prefix" is when running within the
+                associated Python site. The same thing as the "virtualenv root".
+        """
 
-    def __init__(self, hex_version, site_name, prefix):
         self.hex_version = hex_version
         self.site_name = site_name
+        self.site_packages_source = site_packages_source
+        self.mach_site_packages_source = mach_site_packages_source
+        # external_python_executable is needed for commands that tweak the system, such
+        # as "./mach install-moz-phab".
+        self.external_python = external_python
         self.prefix = prefix
 
     def write(self, is_finalized):
         raw = {
             "hex_version": self.hex_version,
             "virtualenv_name": self.site_name,
+            "site_packages_source": self.site_packages_source.name,
+            "mach_site_packages_source": self.mach_site_packages_source.name,
+            "external_python_executable": self.external_python.python_path,
             "is_finalized": is_finalized,
         }
         with open(os.path.join(self.prefix, METADATA_FILENAME), "w") as file:
@@ -55,10 +106,16 @@ class MozSiteMetadata:
             type(self) == type(other)
             and self.hex_version == other.hex_version
             and self.site_name == other.site_name
+            and self.site_packages_source == other.site_packages_source
+            and self.mach_site_packages_source == other.mach_site_packages_source
+            and self.external_python.python_path == other.external_python.python_path
         )
 
     @classmethod
     def from_runtime(cls):
+        if cls.current:
+            return cls.current
+
         return cls.from_path(sys.prefix)
 
     @classmethod
@@ -77,7 +134,10 @@ class MozSiteMetadata:
             return cls(
                 raw["hex_version"],
                 raw["virtualenv_name"],
-                prefix,
+                SitePackagesSource[raw["site_packages_source"]],
+                SitePackagesSource[raw["mach_site_packages_source"]],
+                ExternalPythonSite(raw["external_python_executable"]),
+                metadata_path,
             )
         except FileNotFoundError:
             return None
@@ -85,12 +145,6 @@ class MozSiteMetadata:
             raise out_of_date_exception
 
 
-class SitePackagesSource(enum.Enum):
-    NONE = enum.auto()
-    SYSTEM = enum.auto()
-    VENV = enum.auto()
-
-
 class MachSiteManager:
     """Represents the activate-able "import scope" Mach needs
 
@@ -105,27 +159,45 @@ class MachSiteManager:
 
     def __init__(
         self,
-        topsrcdir,
+        topsrcdir: str,
-        state_dir,
+        state_dir: str,
-        requirements,
+        requirements: MachEnvRequirements,
-        active_metadata,
+        external_python: "ExternalPythonSite",
-        site_packages_source,
+        site_packages_source: SitePackagesSource,
     ):
+        """
+        Args:
+            topsrcdir: The path to the Firefox repo
+            state_dir: The path to the state_dir, generally ~/.mozbuild
+            requirements: The requirements associated with the Mach site, parsed from
+                the file at build/mach_virtualenv_packages.txt
+            external_python: The external Python site that was used to invoke Mach.
+                Usually the system Python, such as /usr/bin/python3
+            site_packages_source: Where the Mach site will import its pip-installed
+                dependencies from
+        """
         self._topsrcdir = topsrcdir
-        self._active_metadata = active_metadata
+        self._external_python = external_python
         self._site_packages_source = site_packages_source
         self._requirements = requirements
         self._virtualenv_root = os.path.join(state_dir, "_virtualenvs", "mach")
         self._metadata = MozSiteMetadata(
             sys.hexversion,
             "mach",
-            self._virtualenv_root,
+            site_packages_source,
+            site_packages_source,
+            external_python,
+            _mach_virtualenv_root(state_dir),
         )
 
     @classmethod
-    def from_environment(
+    def from_environment(cls, topsrcdir: str, state_dir: str):
-        cls, topsrcdir, state_dir, is_mach_create_mach_env_command=False
+        """
-    ):
+        Args:
+            topsrcdir: The path to the Firefox repo
+            state_dir: The path to the state_dir, generally ~/.mozbuild
+        """
+
         requirements = _requirements(topsrcdir, "mach")
         # Mach needs to operate in environments in which no pip packages are installed
         # yet, and the system isn't guaranteed to have the packages we need. For example,
@@ -135,49 +207,42 @@ class MachSiteManager:
             not requirements.pypi_requirements
         ), "Mach pip package requirements must be optional."
 
-        if not (
-            os.environ.get("MACH_USE_SYSTEM_PYTHON") or os.environ.get("MOZ_AUTOMATION")
-        ):
         active_metadata = MozSiteMetadata.from_runtime()
-            if active_metadata and active_metadata.site_name == "mach":
+        if active_metadata:
-                source = SitePackagesSource.VENV
+            external_python = active_metadata.external_python
-            elif not is_mach_create_mach_env_command:
-                # We're in an environment where we normally *would* use the Mach
-                # virtualenv, but we're running a "nativecmd" such as
-                # "create-mach-environment". So, operate Mach without leaning on the
-                # on-disk virtualenv or system packages.
-                source = SitePackagesSource.NONE
-            else:
-                # This catches the case where "./mach create-mach-environment" was run:
-                # * On Mach initialization, the previous clause is hit, and Mach
-                #   operates without any extra site packages
-                # * When the actual "create-mach-environment" function runs,
-                #   MachPythonEnvironment.from_environment() runs again, and this clause
-                #   is triggered. We set the site packages source to the on-disk venv so
-                #   it gets populated.
-                source = SitePackagesSource.VENV
         else:
             external_python = ExternalPythonSite(sys.executable)
-            if external_python.has_pip():
+
-                if os.environ.get("MACH_USE_SYSTEM_PYTHON") and not os.environ.get(
+        if not _system_python_env_variable_present():
-                    "MOZ_AUTOMATION"
+            # We can detect that this is for initializing a native command by:
-                ):
+            # * Checking if we're not using the Mach virtualenv's python, yet the
-                    print(
+            #   "use system python" environment variable isn't set.
-                        "Since Mach has been requested to use the system Python "
+            # * We're still in the "initialization" phase because there's no "current"
-                        "environment, it will need to verify compatibility before "
+            #   MozEnvironment active yet (so, when the `./mach create-mach-environment`
-                        "running the current command. This may take a couple seconds.\n"
+            #   command's function runs, this check will be Falsy).
-                        "Note: you can avoid this delay by unsetting the "
+            # If true, then don't use any "site-packages" sources.
-                        "MACH_USE_SYSTEM_PYTHON environment variable."
+            is_for_initializing_native_cmd = (
+                not active_metadata or active_metadata.site_name != "mach"
+            ) and not MozSiteMetadata.current
+            source = (
+                SitePackagesSource.NONE
+                if is_for_initializing_native_cmd
+                else SitePackagesSource.VENV
             )
-                source = SitePackagesSource.SYSTEM
+        elif not external_python.has_pip():
-            else:
             source = SitePackagesSource.NONE
+        else:
+            source = (
+                SitePackagesSource.SYSTEM
+                if external_python.provides_any_package("mach", requirements)
+                else SitePackagesSource.NONE
+            )
 
         return cls(
             topsrcdir,
             state_dir,
             requirements,
-            MozSiteMetadata.from_runtime(),
+            external_python,
             source,
         )
 
@@ -185,15 +250,18 @@ class MachSiteManager:
         if self._site_packages_source == SitePackagesSource.NONE:
             return True
         elif self._site_packages_source == SitePackagesSource.SYSTEM:
-            _assert_pip_check()
+            pthfile_lines = [
-            external_python = ExternalPythonSite(sys.executable)
+                *self._requirements.pths_as_absolute(self._topsrcdir),
-            external_python.validate_mach_packages(self._requirements)
+                *self._external_python.all_site_packages_dirs(),
+            ]
+            _assert_pip_check(self._topsrcdir, pthfile_lines, "mach")
             return True
         elif self._site_packages_source == SitePackagesSource.VENV:
             environment = self._virtualenv()
             return _is_venv_up_to_date(
                 self._topsrcdir,
                 environment,
+                self._pthfile_lines(environment),
                 self._requirements,
                 self._metadata,
             )
@@ -212,6 +280,8 @@ class MachSiteManager:
         return up_to_date
 
     def activate(self):
+        assert not MozSiteMetadata.current
+
         self.ensure()
         if self._site_packages_source == SitePackagesSource.SYSTEM:
             # Add our Mach modules to the sys.path.
@@ -248,6 +318,8 @@ class MachSiteManager:
                 activate_path = self._virtualenv().activate_path
                 exec(open(activate_path).read(), dict(__file__=activate_path))
 
+        MozSiteMetadata.current = self._metadata
+
     def _build(self):
         if self._site_packages_source != SitePackagesSource.VENV:
             # The Mach virtualenv doesn't have a physical virtualenv on-disk if it won't
@@ -258,51 +330,149 @@ class MachSiteManager:
         _create_venv_with_pthfile(
             self._topsrcdir,
             environment,
+            self._pthfile_lines(environment),
             self._site_packages_source,
             self._requirements,
             self._metadata,
         )
 
+    def _pthfile_lines(self, environment):
+        return [
+            # Prioritize vendored and first-party modules first.
+            *self._requirements.pths_as_absolute(self._topsrcdir),
+            # Then, include the virtualenv's site-packages.
+            *_deprioritize_venv_packages(environment.site_packages_dir()),
+        ]
+
     def _virtualenv(self):
         assert self._site_packages_source == SitePackagesSource.VENV
         return PythonVirtualenv(self._metadata.prefix)
 
 
 class CommandSiteManager:
-    """Public interface for activating virtualenvs and ad-hoc-installing pip packages"""
+    """Activate sites and ad-hoc-install pip packages
+
+    Provides tools to ensure that a command's scope will have expected, compatible
+    packages. Manages prioritization of the import scope, and ensures consistency
+    regardless of how a virtualenv is used (whether via in-process activation, or when
+    used standalone to invoke a script).
+
+    A few notes:
+    * The command environment always inherits Mach's import scope. This is because
+      "unloading" packages in Python is error-prone, so in-process activations will always
+      carry Mach's dependencies along with it. Accordingly, compatibility between each
+      command environment and the Mach environment must be maintained
+    * Unlike the Mach environment, command environments *always* have an associated
+      physical virtualenv on-disk. This is because some commands invoke child Python
+      processes, and that child process should have the same import scope.
+    """
 
     def __init__(
         self,
-        topsrcdir,
+        topsrcdir: str,
-        virtualenvs_dir,
+        state_dir: str,
-        site_name,
+        virtualenv_root: str,
+        site_name: str,
+        active_metadata: MozSiteMetadata,
+        site_packages_source: SitePackagesSource,
+        requirements: MachEnvRequirements,
     ):
-        self.topsrcdir = topsrcdir
+        """
-        self._virtualenv_name = site_name
+        Args:
-        self.virtualenv_root = os.path.join(virtualenvs_dir, site_name)
+            topsrcdir: The path to the Firefox repo
+            state_dir: The path to the state_dir, generally ~/.mozbuild
+            virtualenv_root: The path to the virtualenv associated with this site
+            site_name: The name of this site, such as "build"
+            active_metadata: The currently-active moz-managed site
+            site_packages_source: Where this site will import its pip-installed
+                dependencies from
+            requirements: The requirements associated with this site, parsed from
+                the file at build/<site_name>_virtualenv_packages.txt
+        """
+        self._topsrcdir = topsrcdir
+        self._state_dir = state_dir
+        self.virtualenv_root = virtualenv_root
+        self._site_name = site_name
         self._virtualenv = PythonVirtualenv(self.virtualenv_root)
-        self.bin_path = self._virtualenv.bin_path
         self.python_path = self._virtualenv.python_path
+        self.bin_path = self._virtualenv.bin_path
+        self._site_packages_source = site_packages_source
+        self._mach_site_packages_source = active_metadata.mach_site_packages_source
+        self._external_python = active_metadata.external_python
+        self._requirements = requirements
         self._metadata = MozSiteMetadata(
             sys.hexversion,
             site_name,
-            self._virtualenv.prefix,
+            site_packages_source,
+            active_metadata.mach_site_packages_source,
+            active_metadata.external_python,
+            virtualenv_root,
+        )
+
+    @classmethod
+    def from_environment(
+        cls,
+        topsrcdir: str,
+        state_dir: str,
+        site_name: str,
+        command_virtualenvs_dir: str,
+    ):
+        """
+        Args:
+            topsrcdir: The path to the Firefox repo
+            state_dir: The path to the state_dir, generally ~/.mozbuild
+            site_name: The name of this site, such as "build"
+            command_virtualenvs_dir: The location under which this site's virtualenv
+            should be created
+        """
+        active_metadata = MozSiteMetadata.from_runtime()
+        assert (
+            active_metadata
+        ), "A Mach-managed site must be active before doing work with command sites"
+
+        requirements = _requirements(topsrcdir, site_name)
+        if not _system_python_env_variable_present() or site_name != "build":
+            source = SitePackagesSource.VENV
+        elif not active_metadata.external_python.has_pip():
+            if requirements.pypi_requirements:
+                raise Exception(
+                    f'The "{site_name}" site requires pip '
+                    "packages, and Mach has been told to find such pip packages in "
+                    "the system environment, but it can't because the system doesn't "
+                    'have "pip" installed.'
+                )
+            source = SitePackagesSource.NONE
+        else:
+            source = (
+                SitePackagesSource.SYSTEM
+                if active_metadata.external_python.provides_any_package(
+                    site_name, requirements
+                )
+                else SitePackagesSource.NONE
+            )
+
+        return cls(
+            topsrcdir,
+            state_dir,
+            os.path.join(command_virtualenvs_dir, site_name),
+            site_name,
+            active_metadata,
+            source,
+            requirements,
         )
 
     def ensure(self):
-        """Ensure the site is present and up to date."""
+        """Ensure that this virtualenv is built, up-to-date, and ready for use
-        requirements = _requirements(self.topsrcdir, self._virtualenv_name)
+        If using a virtualenv Python binary directly, it's useful to call this function
-        if not _is_venv_up_to_date(
+        first to ensure that the virtualenv doesn't have obsolete references or packages.
-            self.topsrcdir,
+        """
-            self._virtualenv,
+        if not self._up_to_date():
-            requirements,
-            self._metadata,
-        ):
             _create_venv_with_pthfile(
-                self.topsrcdir,
+                self._topsrcdir,
                 self._virtualenv,
-                SitePackagesSource.VENV,
+                self._pthfile_lines(),
-                requirements,
+                self._site_packages_source,
+                self._requirements,
                 self._metadata,
             )
 
@@ -317,6 +487,7 @@ class CommandSiteManager:
         self.ensure()
         activate_path = self._virtualenv.activate_path
         exec(open(activate_path).read(), dict(__file__=activate_path))
+        MozSiteMetadata.current = self._metadata
 
     def install_pip_package(self, package):
         """Install a package via pip.
@@ -326,7 +497,7 @@ class CommandSiteManager:
 
         If the package is already installed, this is a no-op.
         """
-        if os.path.normcase(sys.executable).startswith(os.path.normcase(self.bin_path)):
+        if Path(sys.prefix) == Path(self.virtualenv_root):
             # If we're already running in this interpreter, we can optimize in
             # the case that the package requirement is already satisfied.
             from pip._internal.req.constructors import install_req_from_line
@@ -350,7 +521,7 @@ class CommandSiteManager:
         """
 
         if not os.path.isabs(path):
-            path = os.path.join(self.topsrcdir, path)
+            path = os.path.join(self._topsrcdir, path)
 
         args = ["--requirement", path]
 
@@ -362,6 +533,83 @@ class CommandSiteManager:
 
         self._virtualenv.pip_install(args)
 
+    def _pthfile_lines(self):
+        """Generate the prioritized import scope to encode in the venv's pthfile
+
+        The import priority looks like this:
+        1. Mach's vendored/first-party modules
+        2. Mach's site-package source (the Mach virtualenv, the system Python, or neither)
+        3. The command's vendored/first-party modules
+        4. The command's site-package source (either the virtualenv or the system Python,
+           if it's not already added)
+
+        Note that, when using the system Python, it may either be prioritized before or
+        after the command's vendored/first-party modules. This is a symptom of us
+        attempting to avoid conflicting with the system packages.
+
+        For example, there's at least one job in CI that operates with an ancient
+        environment with a bunch of old packages, many of whom conflict with our vendored
+        packages. However, the specific command that we're running for the job doesn't
+        need any of the system's packages, so we're safe to insulate ourselves.
+
+        Mach doesn't know the command being run when it's preparing its import scope,
+        so it has to be defensive. Therefore:
+        1. If Mach needs a system package: system packages are higher priority.
+        2. If Mach doesn't need a system package, but the current command does: system
+           packages are still be in the list, albeit at a lower priority.
+        """
+
+        # Prioritize Mach's vendored and first-party modules first.
+        lines = _requirements(self._topsrcdir, "mach").pths_as_absolute(self._topsrcdir)
+        mach_site_packages_source = self._mach_site_packages_source
+        if mach_site_packages_source == SitePackagesSource.SYSTEM:
+            # When Mach is using the system environment, add it next.
+            lines.extend(self._external_python.all_site_packages_dirs())
+        elif mach_site_packages_source == SitePackagesSource.VENV:
+            # When Mach is using its on-disk virtualenv, add its site-packages directory.
+            lines.append(
+                PythonVirtualenv(
+                    _mach_virtualenv_root(self._state_dir)
+                ).site_packages_dir()
+            )
+
+        # Add this command's vendored and first-party modules.
+        lines.extend(self._requirements.pths_as_absolute(self._topsrcdir))
+        if (
+            self._site_packages_source == SitePackagesSource.SYSTEM
+            and not mach_site_packages_source == SitePackagesSource.SYSTEM
+        ):
+            # Only add the system python site-packages if it wasn't already added
+            # earlier for Mach.
+            lines.extend(self._external_python.all_site_packages_dirs())
+        elif self._site_packages_source == SitePackagesSource.VENV:
+            # When allowed to pip install to the on-disk virtualenv, ensure that its
+            # site-packages is in-scope at the end of the list.
+            site_packages_dir = self._virtualenv.site_packages_dir()
+            lines.extend(_deprioritize_venv_packages(site_packages_dir))
+
+        # Note that an on-disk virtualenv is always created for commands, even if they
+        # are using the system as their site-packages source. Also note that, in such
+        # a case, we aren't "deprioritizing" the venv_packages. This is because:
+        # * There should be no risk of breakage, since *nothing* should be "pip install"-d
+        #   into the virtualenv, and
+        # * There's nontrivial complexity in purging the virtualenv's site-packages from
+        #   the sys.path after activation, but since it isn't doing any harm in staying,
+        #   we leave it.
+        return lines
+
+    def _up_to_date(self):
+        if self._site_packages_source == SitePackagesSource.SYSTEM:
+            _assert_pip_check(self._topsrcdir, self._pthfile_lines(), self._site_name)
+
+        return _is_venv_up_to_date(
+            self._topsrcdir,
+            self._virtualenv,
+            self._pthfile_lines(),
+            self._requirements,
+            self._metadata,
+        )
+
 
 class PythonVirtualenv:
     """Calculates paths of interest for general python virtual environments"""
@@ -433,6 +681,7 @@ class ExternalSitePackageValidationResult:
     def __init__(self):
         self._package_discrepancies = []
         self.has_all_packages = True
+        self.provides_any_package = False
 
     def add_discrepancy(self, requirement, found):
         self._package_discrepancies.append((requirement, found))
@@ -475,6 +724,9 @@ class ExternalPythonSite:
                     "import site; print([site.getusersitepackages()] "
                     "+ site.getsitepackages())",
                 ],
+                # The "site" module may return erroneous entries for the system python
+                # if the "VIRTUAL_ENV" environment variable is set.
+                env={k: v for k, v in os.environ.items() if k != "VIRTUAL_ENV"},
                 universal_newlines=True,
             )
             return ast.literal_eval(paths_string)
@@ -488,27 +740,48 @@ class ExternalPythonSite:
             == 0
         )
 
-    def validate_mach_packages(self, requirements):
+    def provides_any_package(self, virtualenv_name, requirements):
         system_packages = self._resolve_installed_packages()
         result = ExternalSitePackageValidationResult()
+        for pkg in requirements.pypi_requirements:
+            installed_version = system_packages.get(pkg.requirement.name)
+            if not installed_version or not pkg.requirement.specifier.contains(
+                installed_version
+            ):
+                result.add_discrepancy(pkg.requirement, installed_version)
+            elif installed_version:
+                result.provides_any_package = True
+
         for pkg in requirements.pypi_optional_requirements:
             installed_version = system_packages.get(pkg.requirement.name)
             if installed_version and not pkg.requirement.specifier.contains(
                 installed_version
             ):
                 result.add_discrepancy(pkg.requirement, installed_version)
+            elif installed_version:
+                result.provides_any_package = True
 
         if not result.has_all_packages:
             print(result.report())
             raise Exception(
-                "Your system Python packages aren't compatible with the "
+                f'The Python packages associated with "{self.python_path}" aren\'t '
-                '"Mach" virtualenv'
+                f'compatible with the "{virtualenv_name}" virtualenv'
             )
 
+        return result.provides_any_package
+
     @functools.lru_cache(maxsize=None)
     def _resolve_installed_packages(self):
         pip_json = subprocess.check_output(
-            [self.python_path, "-m", "pip", "list", "--format", "json"],
+            [
+                self.python_path,
+                "-m",
+                "pip",
+                "list",
+                "--format",
+                "json",
+                "--disable-pip-version-check",
+            ],
             universal_newlines=True,
         )
 
@@ -540,24 +813,114 @@ def _requirements(topsrcdir, virtualenv_name):
     )
 
 
-def _assert_pip_check():
+def _virtualenv_py_path(topsrcdir):
-    """Check that the current site's packages are compatible with each other"""
+    return os.path.join(
-    pip = [sys.executable, "-m", "pip"]
+        topsrcdir, "third_party", "python", "virtualenv", "virtualenv.py"
+    )
+
+
+def _system_python_env_variable_present():
+    return any(
+        os.environ.get(var) for var in ("MACH_USE_SYSTEM_PYTHON", "MOZ_AUTOMATION")
+    )
+
+
+def _assert_pip_check(topsrcdir, pthfile_lines, virtualenv_name):
+    """Check if the provided pthfile lines have a package incompatibility
+
+    If there's an incompatibility, raise an exception and allow it to bubble up since
+    it will require user intervention to resolve.
+    """
+    if os.environ.get(
+        f"MACH_SYSTEM_ASSERTED_COMPATIBLE_WITH_{virtualenv_name.upper()}_SITE", None
+    ):
+        # Don't re-assert compatibility against the system python within Mach subshells.
+        return
+
+    if (
+        virtualenv_name == "mach"
+        and os.environ.get("MACH_USE_SYSTEM_PYTHON")
+        and not os.environ.get("MOZ_AUTOMATION")
+    ):
+        # Since this assertion takes some time, warn users who have explicitly opted
+        # in. Since we only want this message to be printed once, only do it for the
+        # first virtualenv that's used (which is always "mach").
+        print(
+            "Since Mach has been requested to use the system Python "
+            "environment, it will need to verify compatibility before "
+            "running the current command. This may take a couple seconds.\n"
+            "Note: you can avoid this delay by unsetting the "
+            "MACH_USE_SYSTEM_PYTHON environment variable."
+        )
+
+    with tempfile.TemporaryDirectory() as check_env_path:
+        # Pip detects packages on the "sys.path" that have a ".dist-info" or
+        # a ".egg-info" directory. The majority of our Python dependencies are
+        # vendored as extracted wheels or sdists, so they are automatically picked up.
+        # This gives us sufficient confidence to do a `pip check` with both vendored
+        # packages + system packages in scope, and trust the results.
+        # Note: rather than just running the system pip with a modified "sys.path",
+        # we create a new virtualenv that has our pinned pip version, so that
+        # we get consistent results (there's been lots of pip resolver behaviour
+        # changes recently).
+
+        subprocess.check_call(
+            [
+                sys.executable,
+                _virtualenv_py_path(topsrcdir),
+                "--no-download",
+                check_env_path,
+            ],
+            stdout=subprocess.DEVNULL,
+        )
+        check_env = PythonVirtualenv(check_env_path)
+        with open(
+            os.path.join(os.path.join(check_env.site_packages_dir()), PTH_FILENAME),
+            "w",
+        ) as f:
+            f.write("\n".join(pthfile_lines))
+
+        pip = [check_env.python_path, "-m", "pip"]
         check_result = subprocess.run(
-        pip + ["check"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+            pip + ["check"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
         )
         if check_result.returncode:
             print(check_result.stdout, file=sys.stderr)
             subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr)
             raise Exception(
-            'According to "pip check", the current Python site '
+                'According to "pip check", the current Python '
-            f'(associated with "{sys.executable}") has package-compatibility issues.'
+                "environment has package-compatibility issues."
+            )
+
+        os.environ[
+            f"MACH_SYSTEM_ASSERTED_COMPATIBLE_WITH_{virtualenv_name.upper()}_SITE"
+        ] = "1"
+
+
+def _deprioritize_venv_packages(site_packages_dir):
+    # Move the virtualenv's site-packages to the bottom so that vendored packages
+    # are prioritized.
+    # repr(...) is needed to ensure Windows path backslashes aren't mistaken for
+    # escape sequences.
+    # Additionally, when removing the existing "site-packages" entry, we have to
+    # do it in a case-insensitive way because, on Windows:
+    # * Python adds it as <venv>/lib/site-packages
+    # * While distutils tells us it's <venv>/Lib/site-packages
+    # * (note: on-disk, it's capitalized, so distutils is slightly more accurate).
+    return (
+        "import sys; sys.path = [p for p in sys.path if "
+        f"p.lower() != {repr(site_packages_dir)}.lower()]",
+        f"import sys; sys.path.append({repr(site_packages_dir)})",
     )
 
 
 def _create_venv_with_pthfile(
     topsrcdir,
     target_venv,
+    pthfile_lines,
     site_packages_source,
     requirements,
     metadata,
@@ -572,13 +935,7 @@ def _create_venv_with_pthfile(
     subprocess.check_call(
         [
             sys.executable,
-            os.path.join(
+            _virtualenv_py_path(topsrcdir),
-                topsrcdir,
-                "third_party",
-                "python",
-                "virtualenv",
-                "virtualenv.py",
-            ),
             # pip, setuptools and wheel are vendored and inserted into the virtualenv
             # scope automatically, so "virtualenv" doesn't need to seed it.
             "--no-seed",
@@ -589,7 +946,7 @@ def _create_venv_with_pthfile(
     os.utime(target_venv.activate_path, None)
 
     site_packages_dir = target_venv.site_packages_dir()
-    pthfile_contents = "\n".join(requirements.pths_as_absolute(topsrcdir))
+    pthfile_contents = "\n".join(pthfile_lines)
     with open(os.path.join(site_packages_dir, PTH_FILENAME), "w") as f:
         f.write(pthfile_contents)
 
@@ -613,6 +970,7 @@ def _create_venv_with_pthfile(
 def _is_venv_up_to_date(
     topsrcdir,
     target_venv,
+    expected_pthfile_lines,
     requirements,
     expected_metadata,
 ):
@@ -658,8 +1016,12 @@ def _is_venv_up_to_date(
     except FileNotFoundError:
         return False
 
-    expected_pthfile_contents = "\n".join(requirements.pths_as_absolute(topsrcdir))
+    expected_pthfile_contents = "\n".join(expected_pthfile_lines)
     if current_pthfile_contents != expected_pthfile_contents:
         return False
 
     return True
+
+
+def _mach_virtualenv_root(state_dir):
+    return os.path.join(state_dir, "_virtualenvs", "mach")

python/mach/mach/test/test_decorators.py

@@ -5,10 +5,13 @@
 from __future__ import absolute_import, unicode_literals
 
 import os
+from unittest import mock
 
 import pytest
 from unittest.mock import Mock
 
+from mach.requirements import MachEnvRequirements
+from mach.site import CommandSiteManager, SitePackagesSource, MozSiteMetadata
 from mozunit import main
 
 import mach.registrar
@@ -91,6 +94,22 @@ def test_register_command_sets_up_class_at_runtime(registrar):
         )
         inner_function("bar")
 
+    def from_environment_patch(topsrcdir, state_dir, virtualenv_name, dir):
+        return CommandSiteManager(
+            "",
+            "",
+            virtualenv_name,
+            virtualenv_name,
+            MozSiteMetadata(
+                0, "mach", SitePackagesSource.VENV, SitePackagesSource.VENV, "", ""
+            ),
+            SitePackagesSource.VENV,
+            MachEnvRequirements(),
+        )
+
+    with mock.patch.object(
+        CommandSiteManager, "from_environment", from_environment_patch
+    ):
         registrar.dispatch("cmd_foo", context)
         inner_function.assert_called_with("foo")
         registrar.dispatch("cmd_bar", context)

python/mozbuild/mozbuild/base.py

@@ -290,12 +290,14 @@ class MozbuildObject(ProcessExecutionMixin):
     @property
     def virtualenv_manager(self):
         from mach.site import CommandSiteManager
+        from mozboot.util import get_state_dir
 
         if self._virtualenv_manager is None:
-            self._virtualenv_manager = CommandSiteManager(
+            self._virtualenv_manager = CommandSiteManager.from_environment(
                 self.topsrcdir,
-                os.path.join(self.topobjdir, "_virtualenvs"),
+                get_state_dir(),
                 self._virtualenv_name,
+                os.path.join(self.topobjdir, "_virtualenvs"),
             )
 
         return self._virtualenv_manager

python/mozbuild/mozbuild/controller/building.py

@@ -23,13 +23,14 @@ from collections import (
 )
 from textwrap import TextWrapper
 
+from mach.site import CommandSiteManager
+
 try:
     import psutil
 except Exception:
     psutil = None
 
 from mach.mixin.logging import LoggingMixin
-from mozboot.util import get_mach_virtualenv_binary
 import mozfile
 from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
 from mozterm.widgets import Footer
@@ -1508,15 +1509,15 @@ class BuildDriver(MozbuildObject):
                 if eq == "=":
                     append_env[k] = v
 
-        if six.PY3:
+        build_site = CommandSiteManager.from_environment(
-            python = sys.executable
+            self.topsrcdir,
-        else:
+            self.statedir,
-            # Try to get the mach virtualenv Python if we can.
+            "build",
-            python = get_mach_virtualenv_binary()
+            os.path.join(self.topobjdir, "_virtualenvs"),
-            if not os.path.exists(python):
+        )
-                python = "python3"
+        build_site.ensure()
 
-        command = [python, os.path.join(self.topsrcdir, "configure.py")]
+        command = [build_site.python_path, os.path.join(self.topsrcdir, "configure.py")]
         if options:
             command.extend(options)
 

python/mozbuild/mozbuild/mach_commands.py

@@ -2463,7 +2463,6 @@ def create_mach_environment(command_context, force=False):
     manager = MachSiteManager.from_environment(
         command_context.topsrcdir,
         get_state_dir(),
-        is_mach_create_mach_env_command=True,
     )
 
     if manager.ensure(force=force):

python/mozbuild/mozbuild/sphinx.py

@@ -198,6 +198,7 @@ class MozbuildSymbols(Directive):
 
 def setup(app):
     from mach.site import CommandSiteManager
+    from mozboot.util import get_state_dir
     from moztreedocs import manager
 
     app.add_directive("mozbuildsymbols", MozbuildSymbols)
@@ -214,9 +215,10 @@ def setup(app):
     # We need to adjust sys.path in order for Python API docs to get generated
     # properly. We leverage the in-tree virtualenv for this.
     topsrcdir = manager.topsrcdir
-    site = CommandSiteManager(
+    site = CommandSiteManager.from_environment(
         topsrcdir,
-        os.path.join(app.outdir, "_venv"),
+        get_state_dir(),
         "common",
+        os.path.join(app.outdir, "_venv"),
     )
     site.activate()

python/mozbuild/mozbuild/test/test_vendor.py

@@ -17,6 +17,17 @@ import mozunit
 
 def test_up_to_date_vendor():
     with tempfile.TemporaryDirectory() as work_dir:
+
+        def copy_to_work_dir(relative_path):
+            args = (
+                os.path.join(topsrcdir, relative_path),
+                os.path.join(work_dir, relative_path),
+            )
+
+            shutil.copytree(*args) if os.path.isdir(relative_path) else shutil.copy(
+                *args
+            )
+
         subprocess.check_call(["hg", "init", work_dir])
         os.makedirs(os.path.join(work_dir, "build"))
         os.makedirs(os.path.join(work_dir, "third_party"))
@@ -34,17 +45,10 @@ def test_up_to_date_vendor():
             file.write("vendored:third_party/python/setuptools\n")
             file.write("vendored:third_party/python/wheel\n")
 
-        # Copy existing "third_party/python/" vendored files
+        copy_to_work_dir(os.path.join("third_party", "python"))
-        existing_vendored = os.path.join(topsrcdir, "third_party", "python")
+        copy_to_work_dir(os.path.join("python", "mach"))
-        work_vendored = os.path.join(work_dir, "third_party", "python")
+        copy_to_work_dir(os.path.join("build", "mach_virtualenv_packages.txt"))
-        shutil.copytree(existing_vendored, work_vendored)
+        copy_to_work_dir(os.path.join("build", "common_virtualenv_packages.txt"))
-
-        # Copy "mach" module so that `CommandSiteManager` can populate itself.
-        # This is needed because "topsrcdir" is used in this test both for determining
-        # import paths and for acting as a "work dir".
-        existing_mach = os.path.join(topsrcdir, "python", "mach")
-        work_mach = os.path.join(work_dir, "python", "mach")
-        shutil.copytree(existing_mach, work_mach)
 
         # Run the vendoring process
         vendor = VendorPython(
@@ -59,8 +63,8 @@ def test_up_to_date_vendor():
             [
                 "diff",
                 "-r",
-                existing_vendored,
+                os.path.join(topsrcdir, os.path.join("third_party", "python")),
-                work_vendored,
+                os.path.join(work_dir, os.path.join("third_party", "python")),
                 "--exclude=__pycache__",
             ]
         )