Bug 1732948: Assert that the Mach venv isn't out-of-date during init r=ahal

As we leverage the Mach environment more, it becomes increasingly
important that it isn't out-of-date on developer machines.

Add an `up_to_date()` check during Mach initialization.
To minimize the cost to startup, I'm skipping the "pip list" check.

This change required moving `virtualenv` from `mozbuild` to `mach` to
make it available during the early stage of Mach init.

Differential Revision: https://phabricator.services.mozilla.com/D127144

build/docs/python.rst

@@ -73,7 +73,7 @@ to just keep them separate so there is no potential for conflicts.
 Very early in the build process, a virtualenv is created inside the
 :term:`object directory`. The virtualenv is configured such that it can
 find all the Python packages in the source tree. The code for this lives
-in :py:mod:`mozbuild.virtualenv`.
+in :py:mod:`mach.virtualenv`.
 
 Deficiencies
 ------------

build/mach_initialize.py

@@ -173,7 +173,7 @@ def _scrub_system_site_packages():
     sys.path = [path for path in sys.path if path not in site_paths]
 
 
-def _activate_python_environment(topsrcdir):
+def _activate_python_environment(topsrcdir, state_dir):
     # We need the "mach" module to access the logic to parse virtualenv
     # requirements. Since that depends on "packaging" (and, transitively,
     # "pyparsing"), we add those to the path too.
@@ -186,20 +186,17 @@ def _activate_python_environment(topsrcdir):
         )
     ]
 
-    from mach.requirements import MachEnvRequirements
+    from mach.virtualenv import VirtualenvManager
 
-    thunderbird_dir = os.path.join(topsrcdir, "comm")
+    mach_virtualenv = VirtualenvManager(
-    is_thunderbird = os.path.exists(thunderbird_dir) and bool(
-        os.listdir(thunderbird_dir)
-    )
-
-    requirements = MachEnvRequirements.from_requirements_definition(
         topsrcdir,
-        is_thunderbird,
+        os.path.join(state_dir, "_virtualenvs"),
-        True,
+        "mach",
-        os.path.join(topsrcdir, "build", "mach_virtualenv_packages.txt"),
+        populate_local_paths=False,
     )
 
+    requirements = mach_virtualenv.requirements()
+
     if os.environ.get("MACH_USE_SYSTEM_PYTHON") or os.environ.get("MOZ_AUTOMATION"):
         env_var = (
             "MOZ_AUTOMATION"
@@ -257,6 +254,19 @@ def _activate_python_environment(topsrcdir):
         # but we're running a "nativecmd" such as "create-mach-environment".
         # Remove global site packages from sys.path to improve isolation accordingly.
         _scrub_system_site_packages()
+    else:
+        # We're running in the Mach virtualenv - check that it's up-to-date.
+        # Note that the "pip package check" exists to ensure that a virtualenv isn't
+        # corrupted by ad-hoc pip installs. Since the Mach virtualenv is unlikely
+        # to be affected by such installs, and since it takes ~400ms to get the list
+        # of installed pip packages (a *lot* of time to wait during Mach init), we
+        # skip verifying that our pip packages exist.
+        if not mach_virtualenv.up_to_date(skip_pip_package_check=True):
+            print(
+                'The "mach" virtualenv is not up-to-date, please run '
+                '"./mach create-mach-environment"'
+            )
+            sys.exit(1)
 
     sys.path[0:0] = [
         os.path.join(topsrcdir, pth.path)
@@ -288,7 +298,7 @@ def initialize(topsrcdir):
         shutil.rmtree(deleted_dir, ignore_errors=True)
 
     state_dir = _create_state_dir()
-    _activate_python_environment(topsrcdir)
+    _activate_python_environment(topsrcdir, state_dir)
 
     import mach.base
     import mach.main

build/moz.configure/init.configure

@@ -226,8 +226,8 @@ option(env="PYTHON3", nargs=1, help="Python 3 interpreter (3.6 or later)")
 @imports("subprocess")
 @imports("distutils.sysconfig")
 @imports(_from="mozbuild.configure.util", _import="LineIO")
-@imports(_from="mozbuild.virtualenv", _import="VirtualenvManager")
+@imports(_from="mach.virtualenv", _import="VirtualenvManager")
-@imports(_from="mozbuild.virtualenv", _import="verify_python_version")
+@imports(_from="mach.virtualenv", _import="verify_python_version")
 @imports(_from="mozbuild.pythonutil", _import="find_python3_executable")
 @imports(_from="mozbuild.pythonutil", _import="python_executable_version")
 @imports(_from="six", _import="ensure_text")

python/mach/mach/virtualenv.py

@@ -7,6 +7,7 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import functools
 import json
 import os
 import platform
@@ -158,14 +159,12 @@ class VirtualenvManager(VirtualenvHelper):
     def activate_path(self):
         return os.path.join(self.bin_path, "activate_this.py")
 
-    def up_to_date(self):
+    def up_to_date(self, skip_pip_package_check=False):
         """Returns whether the virtualenv is present and up to date.
 
         Args:
-            python: Full path string to the Python executable that this virtualenv
+            skip_pip_package_check: Don't check that the pip state on-disk still meets
-                should be running.  If the Python executable passed in to this
+                our requirements.
-                argument is not the same version as the Python the virtualenv was
-                built with then this method will return False.
         """
 
         # check if virtualenv exists
@@ -174,7 +173,7 @@ class VirtualenvManager(VirtualenvHelper):
         ):
             return False
 
-        env_requirements = self._requirements()
+        env_requirements = self.requirements()
 
         virtualenv_package = os.path.join(
             self.topsrcdir,
@@ -235,10 +234,11 @@ class VirtualenvManager(VirtualenvHelper):
             if current_paths != required_paths:
                 return False
 
-        pip = os.path.join(self.bin_path, "pip")
+        if not skip_pip_package_check:
-        package_result = env_requirements.validate_environment_packages([pip])
+            pip = os.path.join(self.bin_path, "pip")
-        if not package_result.has_all_packages:
+            package_result = env_requirements.validate_environment_packages([pip])
-            return False
+            if not package_result.has_all_packages:
+                return False
 
         return True
 
@@ -304,7 +304,8 @@ class VirtualenvManager(VirtualenvHelper):
         self._disable_pip_outdated_warning()
         return self.virtualenv_root
 
-    def _requirements(self):
+    @functools.lru_cache(maxsize=None)
+    def requirements(self):
         if not os.path.exists(self._manifest_path):
             raise Exception(
                 f'The current command is using the "{self._virtualenv_name}" '
@@ -329,7 +330,7 @@ class VirtualenvManager(VirtualenvHelper):
         This returns the path of the created virtualenv.
         """
         self.create()
-        env_requirements = self._requirements()
+        env_requirements = self.requirements()
         if self.populate_local_paths:
             site_packages_dir = self._site_packages_dir()
             with open(os.path.join(site_packages_dir, PTH_FILENAME), "a") as f:

python/mozboot/mozboot/util.py

@@ -10,7 +10,7 @@ import platform
 import subprocess
 from subprocess import CalledProcessError
 
-from mozbuild.virtualenv import VirtualenvHelper
+from mach.virtualenv import VirtualenvHelper
 from mozfile import which
 
 

python/mozbuild/mozbuild/base.py

@@ -289,7 +289,7 @@ class MozbuildObject(ProcessExecutionMixin):
 
     @property
     def virtualenv_manager(self):
-        from .virtualenv import VirtualenvManager
+        from mach.virtualenv import VirtualenvManager
 
         if self._virtualenv_manager is None:
             self._virtualenv_manager = VirtualenvManager(

python/mozbuild/mozbuild/mach_commands.py

@@ -2445,7 +2445,7 @@ def package_l10n(command_context, verbose=False, locales=[]):
 def create_mach_environment(command_context, force=False):
     """Create the mach virtualenv."""
     from mozboot.util import get_mach_virtualenv_root
-    from mozbuild.virtualenv import VirtualenvManager
+    from mach.virtualenv import VirtualenvManager
 
     virtualenv_path = get_mach_virtualenv_root()
     if sys.executable.startswith(virtualenv_path):

python/mozbuild/mozbuild/sphinx.py

@@ -198,7 +198,7 @@ class MozbuildSymbols(Directive):
 
 
 def setup(app):
-    from mozbuild.virtualenv import VirtualenvManager
+    from mach.virtualenv import VirtualenvManager
     from moztreedocs import manager
 
     app.add_directive("mozbuildsymbols", MozbuildSymbols)