Bug 1723031: In CI, assert Mach pypi package deps using system Python r=ahal

There's an existing algorithm to check if a virtualenv's installed
packages are up-to-date with its requirements. This patch
extracts that logic so that, in cases where we can't automatically
download needed pip packages, we can at least assert that the ones
installed to the system Python are sufficient to meet our requirements.

The current only case in which this system-checking logic is applied
is when starting the Mach virtualenv and the `MOZ_AUTOMATION` or
`MACH_USE_SYSTEM_PYTHON` environment variable is set.

Differential Revision: https://phabricator.services.mozilla.com/D122890

build/mach_initialize.py

@@ -9,6 +9,7 @@ import os
 import platform
 import shutil
 import site
+import subprocess
 import sys
 
 if sys.version_info[0] < 3:
@@ -167,6 +168,11 @@ install a recent enough Python 3.
 """.strip()
 
 
+def _scrub_system_site_packages():
+    site_paths = set(site.getsitepackages() + [site.getusersitepackages()])
+    sys.path = [path for path in sys.path if path not in site_paths]
+
+
 def _activate_python_environment(topsrcdir):
     # We need the "mach" module to access the logic to parse virtualenv
     # requirements. Since that depends on "packaging" (and, transitively,
@@ -193,6 +199,65 @@ def _activate_python_environment(topsrcdir):
         True,
         os.path.join(topsrcdir, "build", "mach_virtualenv_packages.txt"),
     )
+
+    if os.environ.get("MACH_USE_SYSTEM_PYTHON") or os.environ.get("MOZ_AUTOMATION"):
+        env_var = (
+            "MOZ_AUTOMATION"
+            if os.environ.get("MOZ_AUTOMATION")
+            else "MACH_USE_SYSTEM_PYTHON"
+        )
+
+        has_pip = (
+            subprocess.run(
+                [sys.executable, "-c", "import pip"], stderr=subprocess.DEVNULL
+            ).returncode
+            == 0
+        )
+        # There are environments in CI that aren't prepared to provide any Mach dependency
+        # packages. Changing this is a nontrivial endeavour, so guard against having
+        # non-optional Mach requirements.
+        assert (
+            not requirements.pypi_requirements
+        ), "Mach pip package requirements must be optional."
+        if has_pip:
+            pip = [sys.executable, "-m", "pip"]
+            check_result = subprocess.run(
+                pip + ["check"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                universal_newlines=True,
+            )
+            if check_result.returncode:
+                print(check_result.stdout, file=sys.stderr)
+                subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr)
+                raise Exception(
+                    'According to "pip check", the current Python '
+                    "environment has package-compatibility issues."
+                )
+
+            package_result = requirements.validate_environment_packages(pip)
+            if not package_result.has_all_packages:
+                print(
+                    "Skipping automatic management of Python dependencies since "
+                    f"the '{env_var}' environment variable is set.\n"
+                    "The following issues were found while validating your Python "
+                    "environment:"
+                )
+                print(package_result.report())
+                sys.exit(1)
+        else:
+            # Pip isn't installed to the system Python environment, so we can't use
+            # it to verify compatibility with Mach. Remove the system site-packages
+            # from the import scope so that Mach behaves as though all of its
+            # (optional) dependencies are not installed.
+            _scrub_system_site_packages()
+
+    elif sys.prefix == sys.base_prefix:
+        # We're in an environment where we normally use the Mach virtualenv,
+        # but we're running a "nativecmd" such as "create-mach-environment".
+        # Remove global site packages from sys.path to improve isolation accordingly.
+        _scrub_system_site_packages()
+
     sys.path[0:0] = [
         os.path.join(topsrcdir, pth.path)
         for pth in requirements.pth_requirements + requirements.vendored_requirements
@@ -222,12 +287,6 @@ def initialize(topsrcdir):
     if os.path.exists(deleted_dir):
         shutil.rmtree(deleted_dir, ignore_errors=True)
 
-    if sys.prefix == sys.base_prefix:
-        # We are not in a virtualenv. Remove global site packages
-        # from sys.path.
-        site_paths = set(site.getsitepackages() + [site.getusersitepackages()])
-        sys.path = [path for path in sys.path if path not in site_paths]
-
     state_dir = _create_state_dir()
     _activate_python_environment(topsrcdir)