diff --git a/configure.py b/configure.py index 2d420dceff46..b533876812be 100644 --- a/configure.py +++ b/configure.py @@ -22,7 +22,8 @@ sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild")) sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging")) sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing")) sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six")) -from mach.site import CommandSiteManager +from mach.site import CommandSiteManager, MachSiteManager, MozSiteMetadata +from mozboot.util import get_state_dir from mozbuild.configure import ( ConfigureSandbox, TRACE, @@ -226,16 +227,38 @@ def config_status(config, execute=True): def _activate_build_virtualenv(): + """Ensure that the build virtualenv is activated + + configure.py may be executed through Mach, or via "./configure, make". + In the first case, the build virtualenv should already be activated. + In the second case, we're likely being executed with the system Python, and must + prepare the virtualenv and activate it ourselves. + """ + version = ".".join(str(i) for i in sys.version_info[0:3]) print(f"Using Python {version} from {sys.executable}") + active_site = MozSiteMetadata.from_runtime() + if active_site and active_site.site_name == "build": + # We're already running within the "build" virtualenv, no additional work is + # needed. + return + + # We're using the system python (or are nested within a non-build mach-managed + # virtualenv), so we should activate the build virtualenv as expected by the rest of + # configure. + topobjdir = os.path.realpath(".") topsrcdir = os.path.realpath(os.path.dirname(__file__)) + state_dir = get_state_dir() - build_site = CommandSiteManager( + mach_site = MachSiteManager.from_environment(topsrcdir, state_dir) + mach_site.activate() + build_site = CommandSiteManager.from_environment( topsrcdir, - os.path.join(topobjdir, "_virtualenvs"), + state_dir, "build", + os.path.join(topobjdir, "_virtualenvs"), ) if not build_site.ensure(): print("Created Python 3 virtualenv") diff --git a/python/mach/mach/site.py b/python/mach/mach/site.py index 6333b378688b..29f3ab1f0226 100644 --- a/python/mach/mach/site.py +++ b/python/mach/mach/site.py @@ -18,8 +18,12 @@ import site import subprocess import sys from pathlib import Path +import tempfile +from typing import Optional -from mach.requirements import MachEnvRequirements +from mach.requirements import ( + MachEnvRequirements, +) PTH_FILENAME = "mach.pth" METADATA_FILENAME = "moz_virtualenv_metadata.json" @@ -33,18 +37,65 @@ class MozSiteMetadataOutOfDateError(Exception): pass -class MozSiteMetadata: - """Details about a Moz-managed python environment""" +class SitePackagesSource(enum.Enum): + NONE = enum.auto() + SYSTEM = enum.auto() + VENV = enum.auto() + + +class MozSiteMetadata: + """Details about a Moz-managed python site + + When a Moz-managed site is active, its associated metadata is available + at "MozSiteMetadata.current". + + Sites that have associated virtualenvs (so, those that aren't strictly leaning on + the external python packages) will have their metadata written to + /moz_virtualenv_metadata.json. + """ + + # Used to track which which virtualenv has been activated in-process. + current: Optional["MozSiteMetadata"] = None + + def __init__( + self, + hex_version: int, + site_name: str, + site_packages_source: SitePackagesSource, + mach_site_packages_source: SitePackagesSource, + external_python: "ExternalPythonSite", + prefix: str, + ): + """ + Args: + hex_version: The python version number from sys.hexversion + site_name: The name of the site this metadata is associated with + site_packages_source: Where this site imports its + pip-installed dependencies from + mach_site_packages_source: Where the Mach site imports + its pip-installed dependencies from + external_python: The external Python site that was + used to invoke Mach. Usually the system Python, such as /usr/bin/python3 + prefix: The same value as "sys.prefix" is when running within the + associated Python site. The same thing as the "virtualenv root". + """ - def __init__(self, hex_version, site_name, prefix): self.hex_version = hex_version self.site_name = site_name + self.site_packages_source = site_packages_source + self.mach_site_packages_source = mach_site_packages_source + # external_python_executable is needed for commands that tweak the system, such + # as "./mach install-moz-phab". + self.external_python = external_python self.prefix = prefix def write(self, is_finalized): raw = { "hex_version": self.hex_version, "virtualenv_name": self.site_name, + "site_packages_source": self.site_packages_source.name, + "mach_site_packages_source": self.mach_site_packages_source.name, + "external_python_executable": self.external_python.python_path, "is_finalized": is_finalized, } with open(os.path.join(self.prefix, METADATA_FILENAME), "w") as file: @@ -55,10 +106,16 @@ class MozSiteMetadata: type(self) == type(other) and self.hex_version == other.hex_version and self.site_name == other.site_name + and self.site_packages_source == other.site_packages_source + and self.mach_site_packages_source == other.mach_site_packages_source + and self.external_python.python_path == other.external_python.python_path ) @classmethod def from_runtime(cls): + if cls.current: + return cls.current + return cls.from_path(sys.prefix) @classmethod @@ -77,7 +134,10 @@ class MozSiteMetadata: return cls( raw["hex_version"], raw["virtualenv_name"], - prefix, + SitePackagesSource[raw["site_packages_source"]], + SitePackagesSource[raw["mach_site_packages_source"]], + ExternalPythonSite(raw["external_python_executable"]), + metadata_path, ) except FileNotFoundError: return None @@ -85,12 +145,6 @@ class MozSiteMetadata: raise out_of_date_exception -class SitePackagesSource(enum.Enum): - NONE = enum.auto() - SYSTEM = enum.auto() - VENV = enum.auto() - - class MachSiteManager: """Represents the activate-able "import scope" Mach needs @@ -105,27 +159,45 @@ class MachSiteManager: def __init__( self, - topsrcdir, - state_dir, - requirements, - active_metadata, - site_packages_source, + topsrcdir: str, + state_dir: str, + requirements: MachEnvRequirements, + external_python: "ExternalPythonSite", + site_packages_source: SitePackagesSource, ): + """ + Args: + topsrcdir: The path to the Firefox repo + state_dir: The path to the state_dir, generally ~/.mozbuild + requirements: The requirements associated with the Mach site, parsed from + the file at build/mach_virtualenv_packages.txt + external_python: The external Python site that was used to invoke Mach. + Usually the system Python, such as /usr/bin/python3 + site_packages_source: Where the Mach site will import its pip-installed + dependencies from + """ self._topsrcdir = topsrcdir - self._active_metadata = active_metadata + self._external_python = external_python self._site_packages_source = site_packages_source self._requirements = requirements self._virtualenv_root = os.path.join(state_dir, "_virtualenvs", "mach") self._metadata = MozSiteMetadata( sys.hexversion, "mach", - self._virtualenv_root, + site_packages_source, + site_packages_source, + external_python, + _mach_virtualenv_root(state_dir), ) @classmethod - def from_environment( - cls, topsrcdir, state_dir, is_mach_create_mach_env_command=False - ): + def from_environment(cls, topsrcdir: str, state_dir: str): + """ + Args: + topsrcdir: The path to the Firefox repo + state_dir: The path to the state_dir, generally ~/.mozbuild + """ + requirements = _requirements(topsrcdir, "mach") # Mach needs to operate in environments in which no pip packages are installed # yet, and the system isn't guaranteed to have the packages we need. For example, @@ -135,49 +207,42 @@ class MachSiteManager: not requirements.pypi_requirements ), "Mach pip package requirements must be optional." - if not ( - os.environ.get("MACH_USE_SYSTEM_PYTHON") or os.environ.get("MOZ_AUTOMATION") - ): - active_metadata = MozSiteMetadata.from_runtime() - if active_metadata and active_metadata.site_name == "mach": - source = SitePackagesSource.VENV - elif not is_mach_create_mach_env_command: - # We're in an environment where we normally *would* use the Mach - # virtualenv, but we're running a "nativecmd" such as - # "create-mach-environment". So, operate Mach without leaning on the - # on-disk virtualenv or system packages. - source = SitePackagesSource.NONE - else: - # This catches the case where "./mach create-mach-environment" was run: - # * On Mach initialization, the previous clause is hit, and Mach - # operates without any extra site packages - # * When the actual "create-mach-environment" function runs, - # MachPythonEnvironment.from_environment() runs again, and this clause - # is triggered. We set the site packages source to the on-disk venv so - # it gets populated. - source = SitePackagesSource.VENV + active_metadata = MozSiteMetadata.from_runtime() + if active_metadata: + external_python = active_metadata.external_python else: external_python = ExternalPythonSite(sys.executable) - if external_python.has_pip(): - if os.environ.get("MACH_USE_SYSTEM_PYTHON") and not os.environ.get( - "MOZ_AUTOMATION" - ): - print( - "Since Mach has been requested to use the system Python " - "environment, it will need to verify compatibility before " - "running the current command. This may take a couple seconds.\n" - "Note: you can avoid this delay by unsetting the " - "MACH_USE_SYSTEM_PYTHON environment variable." - ) - source = SitePackagesSource.SYSTEM - else: - source = SitePackagesSource.NONE + + if not _system_python_env_variable_present(): + # We can detect that this is for initializing a native command by: + # * Checking if we're not using the Mach virtualenv's python, yet the + # "use system python" environment variable isn't set. + # * We're still in the "initialization" phase because there's no "current" + # MozEnvironment active yet (so, when the `./mach create-mach-environment` + # command's function runs, this check will be Falsy). + # If true, then don't use any "site-packages" sources. + is_for_initializing_native_cmd = ( + not active_metadata or active_metadata.site_name != "mach" + ) and not MozSiteMetadata.current + source = ( + SitePackagesSource.NONE + if is_for_initializing_native_cmd + else SitePackagesSource.VENV + ) + elif not external_python.has_pip(): + source = SitePackagesSource.NONE + else: + source = ( + SitePackagesSource.SYSTEM + if external_python.provides_any_package("mach", requirements) + else SitePackagesSource.NONE + ) return cls( topsrcdir, state_dir, requirements, - MozSiteMetadata.from_runtime(), + external_python, source, ) @@ -185,15 +250,18 @@ class MachSiteManager: if self._site_packages_source == SitePackagesSource.NONE: return True elif self._site_packages_source == SitePackagesSource.SYSTEM: - _assert_pip_check() - external_python = ExternalPythonSite(sys.executable) - external_python.validate_mach_packages(self._requirements) + pthfile_lines = [ + *self._requirements.pths_as_absolute(self._topsrcdir), + *self._external_python.all_site_packages_dirs(), + ] + _assert_pip_check(self._topsrcdir, pthfile_lines, "mach") return True elif self._site_packages_source == SitePackagesSource.VENV: environment = self._virtualenv() return _is_venv_up_to_date( self._topsrcdir, environment, + self._pthfile_lines(environment), self._requirements, self._metadata, ) @@ -212,6 +280,8 @@ class MachSiteManager: return up_to_date def activate(self): + assert not MozSiteMetadata.current + self.ensure() if self._site_packages_source == SitePackagesSource.SYSTEM: # Add our Mach modules to the sys.path. @@ -248,6 +318,8 @@ class MachSiteManager: activate_path = self._virtualenv().activate_path exec(open(activate_path).read(), dict(__file__=activate_path)) + MozSiteMetadata.current = self._metadata + def _build(self): if self._site_packages_source != SitePackagesSource.VENV: # The Mach virtualenv doesn't have a physical virtualenv on-disk if it won't @@ -258,51 +330,149 @@ class MachSiteManager: _create_venv_with_pthfile( self._topsrcdir, environment, + self._pthfile_lines(environment), self._site_packages_source, self._requirements, self._metadata, ) + def _pthfile_lines(self, environment): + return [ + # Prioritize vendored and first-party modules first. + *self._requirements.pths_as_absolute(self._topsrcdir), + # Then, include the virtualenv's site-packages. + *_deprioritize_venv_packages(environment.site_packages_dir()), + ] + def _virtualenv(self): assert self._site_packages_source == SitePackagesSource.VENV return PythonVirtualenv(self._metadata.prefix) class CommandSiteManager: - """Public interface for activating virtualenvs and ad-hoc-installing pip packages""" + """Activate sites and ad-hoc-install pip packages + + Provides tools to ensure that a command's scope will have expected, compatible + packages. Manages prioritization of the import scope, and ensures consistency + regardless of how a virtualenv is used (whether via in-process activation, or when + used standalone to invoke a script). + + A few notes: + * The command environment always inherits Mach's import scope. This is because + "unloading" packages in Python is error-prone, so in-process activations will always + carry Mach's dependencies along with it. Accordingly, compatibility between each + command environment and the Mach environment must be maintained + * Unlike the Mach environment, command environments *always* have an associated + physical virtualenv on-disk. This is because some commands invoke child Python + processes, and that child process should have the same import scope. + """ def __init__( self, - topsrcdir, - virtualenvs_dir, - site_name, + topsrcdir: str, + state_dir: str, + virtualenv_root: str, + site_name: str, + active_metadata: MozSiteMetadata, + site_packages_source: SitePackagesSource, + requirements: MachEnvRequirements, ): - self.topsrcdir = topsrcdir - self._virtualenv_name = site_name - self.virtualenv_root = os.path.join(virtualenvs_dir, site_name) + """ + Args: + topsrcdir: The path to the Firefox repo + state_dir: The path to the state_dir, generally ~/.mozbuild + virtualenv_root: The path to the virtualenv associated with this site + site_name: The name of this site, such as "build" + active_metadata: The currently-active moz-managed site + site_packages_source: Where this site will import its pip-installed + dependencies from + requirements: The requirements associated with this site, parsed from + the file at build/_virtualenv_packages.txt + """ + self._topsrcdir = topsrcdir + self._state_dir = state_dir + self.virtualenv_root = virtualenv_root + self._site_name = site_name self._virtualenv = PythonVirtualenv(self.virtualenv_root) - self.bin_path = self._virtualenv.bin_path self.python_path = self._virtualenv.python_path + self.bin_path = self._virtualenv.bin_path + self._site_packages_source = site_packages_source + self._mach_site_packages_source = active_metadata.mach_site_packages_source + self._external_python = active_metadata.external_python + self._requirements = requirements self._metadata = MozSiteMetadata( sys.hexversion, site_name, - self._virtualenv.prefix, + site_packages_source, + active_metadata.mach_site_packages_source, + active_metadata.external_python, + virtualenv_root, + ) + + @classmethod + def from_environment( + cls, + topsrcdir: str, + state_dir: str, + site_name: str, + command_virtualenvs_dir: str, + ): + """ + Args: + topsrcdir: The path to the Firefox repo + state_dir: The path to the state_dir, generally ~/.mozbuild + site_name: The name of this site, such as "build" + command_virtualenvs_dir: The location under which this site's virtualenv + should be created + """ + active_metadata = MozSiteMetadata.from_runtime() + assert ( + active_metadata + ), "A Mach-managed site must be active before doing work with command sites" + + requirements = _requirements(topsrcdir, site_name) + if not _system_python_env_variable_present() or site_name != "build": + source = SitePackagesSource.VENV + elif not active_metadata.external_python.has_pip(): + if requirements.pypi_requirements: + raise Exception( + f'The "{site_name}" site requires pip ' + "packages, and Mach has been told to find such pip packages in " + "the system environment, but it can't because the system doesn't " + 'have "pip" installed.' + ) + source = SitePackagesSource.NONE + else: + source = ( + SitePackagesSource.SYSTEM + if active_metadata.external_python.provides_any_package( + site_name, requirements + ) + else SitePackagesSource.NONE + ) + + return cls( + topsrcdir, + state_dir, + os.path.join(command_virtualenvs_dir, site_name), + site_name, + active_metadata, + source, + requirements, ) def ensure(self): - """Ensure the site is present and up to date.""" - requirements = _requirements(self.topsrcdir, self._virtualenv_name) - if not _is_venv_up_to_date( - self.topsrcdir, - self._virtualenv, - requirements, - self._metadata, - ): + """Ensure that this virtualenv is built, up-to-date, and ready for use + If using a virtualenv Python binary directly, it's useful to call this function + first to ensure that the virtualenv doesn't have obsolete references or packages. + """ + if not self._up_to_date(): _create_venv_with_pthfile( - self.topsrcdir, + self._topsrcdir, self._virtualenv, - SitePackagesSource.VENV, - requirements, + self._pthfile_lines(), + self._site_packages_source, + self._requirements, self._metadata, ) @@ -317,6 +487,7 @@ class CommandSiteManager: self.ensure() activate_path = self._virtualenv.activate_path exec(open(activate_path).read(), dict(__file__=activate_path)) + MozSiteMetadata.current = self._metadata def install_pip_package(self, package): """Install a package via pip. @@ -326,7 +497,7 @@ class CommandSiteManager: If the package is already installed, this is a no-op. """ - if os.path.normcase(sys.executable).startswith(os.path.normcase(self.bin_path)): + if Path(sys.prefix) == Path(self.virtualenv_root): # If we're already running in this interpreter, we can optimize in # the case that the package requirement is already satisfied. from pip._internal.req.constructors import install_req_from_line @@ -350,7 +521,7 @@ class CommandSiteManager: """ if not os.path.isabs(path): - path = os.path.join(self.topsrcdir, path) + path = os.path.join(self._topsrcdir, path) args = ["--requirement", path] @@ -362,6 +533,83 @@ class CommandSiteManager: self._virtualenv.pip_install(args) + def _pthfile_lines(self): + """Generate the prioritized import scope to encode in the venv's pthfile + + The import priority looks like this: + 1. Mach's vendored/first-party modules + 2. Mach's site-package source (the Mach virtualenv, the system Python, or neither) + 3. The command's vendored/first-party modules + 4. The command's site-package source (either the virtualenv or the system Python, + if it's not already added) + + Note that, when using the system Python, it may either be prioritized before or + after the command's vendored/first-party modules. This is a symptom of us + attempting to avoid conflicting with the system packages. + + For example, there's at least one job in CI that operates with an ancient + environment with a bunch of old packages, many of whom conflict with our vendored + packages. However, the specific command that we're running for the job doesn't + need any of the system's packages, so we're safe to insulate ourselves. + + Mach doesn't know the command being run when it's preparing its import scope, + so it has to be defensive. Therefore: + 1. If Mach needs a system package: system packages are higher priority. + 2. If Mach doesn't need a system package, but the current command does: system + packages are still be in the list, albeit at a lower priority. + """ + + # Prioritize Mach's vendored and first-party modules first. + lines = _requirements(self._topsrcdir, "mach").pths_as_absolute(self._topsrcdir) + mach_site_packages_source = self._mach_site_packages_source + if mach_site_packages_source == SitePackagesSource.SYSTEM: + # When Mach is using the system environment, add it next. + lines.extend(self._external_python.all_site_packages_dirs()) + elif mach_site_packages_source == SitePackagesSource.VENV: + # When Mach is using its on-disk virtualenv, add its site-packages directory. + lines.append( + PythonVirtualenv( + _mach_virtualenv_root(self._state_dir) + ).site_packages_dir() + ) + + # Add this command's vendored and first-party modules. + lines.extend(self._requirements.pths_as_absolute(self._topsrcdir)) + if ( + self._site_packages_source == SitePackagesSource.SYSTEM + and not mach_site_packages_source == SitePackagesSource.SYSTEM + ): + # Only add the system python site-packages if it wasn't already added + # earlier for Mach. + lines.extend(self._external_python.all_site_packages_dirs()) + elif self._site_packages_source == SitePackagesSource.VENV: + # When allowed to pip install to the on-disk virtualenv, ensure that its + # site-packages is in-scope at the end of the list. + site_packages_dir = self._virtualenv.site_packages_dir() + lines.extend(_deprioritize_venv_packages(site_packages_dir)) + + # Note that an on-disk virtualenv is always created for commands, even if they + # are using the system as their site-packages source. Also note that, in such + # a case, we aren't "deprioritizing" the venv_packages. This is because: + # * There should be no risk of breakage, since *nothing* should be "pip install"-d + # into the virtualenv, and + # * There's nontrivial complexity in purging the virtualenv's site-packages from + # the sys.path after activation, but since it isn't doing any harm in staying, + # we leave it. + return lines + + def _up_to_date(self): + if self._site_packages_source == SitePackagesSource.SYSTEM: + _assert_pip_check(self._topsrcdir, self._pthfile_lines(), self._site_name) + + return _is_venv_up_to_date( + self._topsrcdir, + self._virtualenv, + self._pthfile_lines(), + self._requirements, + self._metadata, + ) + class PythonVirtualenv: """Calculates paths of interest for general python virtual environments""" @@ -433,6 +681,7 @@ class ExternalSitePackageValidationResult: def __init__(self): self._package_discrepancies = [] self.has_all_packages = True + self.provides_any_package = False def add_discrepancy(self, requirement, found): self._package_discrepancies.append((requirement, found)) @@ -475,6 +724,9 @@ class ExternalPythonSite: "import site; print([site.getusersitepackages()] " "+ site.getsitepackages())", ], + # The "site" module may return erroneous entries for the system python + # if the "VIRTUAL_ENV" environment variable is set. + env={k: v for k, v in os.environ.items() if k != "VIRTUAL_ENV"}, universal_newlines=True, ) return ast.literal_eval(paths_string) @@ -488,27 +740,48 @@ class ExternalPythonSite: == 0 ) - def validate_mach_packages(self, requirements): + def provides_any_package(self, virtualenv_name, requirements): system_packages = self._resolve_installed_packages() result = ExternalSitePackageValidationResult() + for pkg in requirements.pypi_requirements: + installed_version = system_packages.get(pkg.requirement.name) + if not installed_version or not pkg.requirement.specifier.contains( + installed_version + ): + result.add_discrepancy(pkg.requirement, installed_version) + elif installed_version: + result.provides_any_package = True + for pkg in requirements.pypi_optional_requirements: installed_version = system_packages.get(pkg.requirement.name) if installed_version and not pkg.requirement.specifier.contains( installed_version ): result.add_discrepancy(pkg.requirement, installed_version) + elif installed_version: + result.provides_any_package = True if not result.has_all_packages: print(result.report()) raise Exception( - "Your system Python packages aren't compatible with the " - '"Mach" virtualenv' + f'The Python packages associated with "{self.python_path}" aren\'t ' + f'compatible with the "{virtualenv_name}" virtualenv' ) + return result.provides_any_package + @functools.lru_cache(maxsize=None) def _resolve_installed_packages(self): pip_json = subprocess.check_output( - [self.python_path, "-m", "pip", "list", "--format", "json"], + [ + self.python_path, + "-m", + "pip", + "list", + "--format", + "json", + "--disable-pip-version-check", + ], universal_newlines=True, ) @@ -540,24 +813,114 @@ def _requirements(topsrcdir, virtualenv_name): ) -def _assert_pip_check(): - """Check that the current site's packages are compatible with each other""" - pip = [sys.executable, "-m", "pip"] - check_result = subprocess.run( - pip + ["check"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT +def _virtualenv_py_path(topsrcdir): + return os.path.join( + topsrcdir, "third_party", "python", "virtualenv", "virtualenv.py" ) - if check_result.returncode: - print(check_result.stdout, file=sys.stderr) - subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr) - raise Exception( - 'According to "pip check", the current Python site ' - f'(associated with "{sys.executable}") has package-compatibility issues.' + + +def _system_python_env_variable_present(): + return any( + os.environ.get(var) for var in ("MACH_USE_SYSTEM_PYTHON", "MOZ_AUTOMATION") + ) + + +def _assert_pip_check(topsrcdir, pthfile_lines, virtualenv_name): + """Check if the provided pthfile lines have a package incompatibility + + If there's an incompatibility, raise an exception and allow it to bubble up since + it will require user intervention to resolve. + """ + if os.environ.get( + f"MACH_SYSTEM_ASSERTED_COMPATIBLE_WITH_{virtualenv_name.upper()}_SITE", None + ): + # Don't re-assert compatibility against the system python within Mach subshells. + return + + if ( + virtualenv_name == "mach" + and os.environ.get("MACH_USE_SYSTEM_PYTHON") + and not os.environ.get("MOZ_AUTOMATION") + ): + # Since this assertion takes some time, warn users who have explicitly opted + # in. Since we only want this message to be printed once, only do it for the + # first virtualenv that's used (which is always "mach"). + print( + "Since Mach has been requested to use the system Python " + "environment, it will need to verify compatibility before " + "running the current command. This may take a couple seconds.\n" + "Note: you can avoid this delay by unsetting the " + "MACH_USE_SYSTEM_PYTHON environment variable." ) + with tempfile.TemporaryDirectory() as check_env_path: + # Pip detects packages on the "sys.path" that have a ".dist-info" or + # a ".egg-info" directory. The majority of our Python dependencies are + # vendored as extracted wheels or sdists, so they are automatically picked up. + # This gives us sufficient confidence to do a `pip check` with both vendored + # packages + system packages in scope, and trust the results. + # Note: rather than just running the system pip with a modified "sys.path", + # we create a new virtualenv that has our pinned pip version, so that + # we get consistent results (there's been lots of pip resolver behaviour + # changes recently). + + subprocess.check_call( + [ + sys.executable, + _virtualenv_py_path(topsrcdir), + "--no-download", + check_env_path, + ], + stdout=subprocess.DEVNULL, + ) + check_env = PythonVirtualenv(check_env_path) + with open( + os.path.join(os.path.join(check_env.site_packages_dir()), PTH_FILENAME), + "w", + ) as f: + f.write("\n".join(pthfile_lines)) + + pip = [check_env.python_path, "-m", "pip"] + check_result = subprocess.run( + pip + ["check"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + if check_result.returncode: + print(check_result.stdout, file=sys.stderr) + subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr) + raise Exception( + 'According to "pip check", the current Python ' + "environment has package-compatibility issues." + ) + + os.environ[ + f"MACH_SYSTEM_ASSERTED_COMPATIBLE_WITH_{virtualenv_name.upper()}_SITE" + ] = "1" + + +def _deprioritize_venv_packages(site_packages_dir): + # Move the virtualenv's site-packages to the bottom so that vendored packages + # are prioritized. + # repr(...) is needed to ensure Windows path backslashes aren't mistaken for + # escape sequences. + # Additionally, when removing the existing "site-packages" entry, we have to + # do it in a case-insensitive way because, on Windows: + # * Python adds it as /lib/site-packages + # * While distutils tells us it's /Lib/site-packages + # * (note: on-disk, it's capitalized, so distutils is slightly more accurate). + return ( + "import sys; sys.path = [p for p in sys.path if " + f"p.lower() != {repr(site_packages_dir)}.lower()]", + f"import sys; sys.path.append({repr(site_packages_dir)})", + ) + def _create_venv_with_pthfile( topsrcdir, target_venv, + pthfile_lines, site_packages_source, requirements, metadata, @@ -572,13 +935,7 @@ def _create_venv_with_pthfile( subprocess.check_call( [ sys.executable, - os.path.join( - topsrcdir, - "third_party", - "python", - "virtualenv", - "virtualenv.py", - ), + _virtualenv_py_path(topsrcdir), # pip, setuptools and wheel are vendored and inserted into the virtualenv # scope automatically, so "virtualenv" doesn't need to seed it. "--no-seed", @@ -589,7 +946,7 @@ def _create_venv_with_pthfile( os.utime(target_venv.activate_path, None) site_packages_dir = target_venv.site_packages_dir() - pthfile_contents = "\n".join(requirements.pths_as_absolute(topsrcdir)) + pthfile_contents = "\n".join(pthfile_lines) with open(os.path.join(site_packages_dir, PTH_FILENAME), "w") as f: f.write(pthfile_contents) @@ -613,6 +970,7 @@ def _create_venv_with_pthfile( def _is_venv_up_to_date( topsrcdir, target_venv, + expected_pthfile_lines, requirements, expected_metadata, ): @@ -658,8 +1016,12 @@ def _is_venv_up_to_date( except FileNotFoundError: return False - expected_pthfile_contents = "\n".join(requirements.pths_as_absolute(topsrcdir)) + expected_pthfile_contents = "\n".join(expected_pthfile_lines) if current_pthfile_contents != expected_pthfile_contents: return False return True + + +def _mach_virtualenv_root(state_dir): + return os.path.join(state_dir, "_virtualenvs", "mach") diff --git a/python/mach/mach/test/test_decorators.py b/python/mach/mach/test/test_decorators.py index 9b8f20921dc4..84b2d3f3d88b 100644 --- a/python/mach/mach/test/test_decorators.py +++ b/python/mach/mach/test/test_decorators.py @@ -5,10 +5,13 @@ from __future__ import absolute_import, unicode_literals import os +from unittest import mock import pytest from unittest.mock import Mock +from mach.requirements import MachEnvRequirements +from mach.site import CommandSiteManager, SitePackagesSource, MozSiteMetadata from mozunit import main import mach.registrar @@ -91,10 +94,26 @@ def test_register_command_sets_up_class_at_runtime(registrar): ) inner_function("bar") - registrar.dispatch("cmd_foo", context) - inner_function.assert_called_with("foo") - registrar.dispatch("cmd_bar", context) - inner_function.assert_called_with("bar") + def from_environment_patch(topsrcdir, state_dir, virtualenv_name, dir): + return CommandSiteManager( + "", + "", + virtualenv_name, + virtualenv_name, + MozSiteMetadata( + 0, "mach", SitePackagesSource.VENV, SitePackagesSource.VENV, "", "" + ), + SitePackagesSource.VENV, + MachEnvRequirements(), + ) + + with mock.patch.object( + CommandSiteManager, "from_environment", from_environment_patch + ): + registrar.dispatch("cmd_foo", context) + inner_function.assert_called_with("foo") + registrar.dispatch("cmd_bar", context) + inner_function.assert_called_with("bar") def test_cannot_create_command_nonexisting_category(registrar): diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py index 6832efea2b5a..118fbba9af2f 100644 --- a/python/mozbuild/mozbuild/base.py +++ b/python/mozbuild/mozbuild/base.py @@ -290,12 +290,14 @@ class MozbuildObject(ProcessExecutionMixin): @property def virtualenv_manager(self): from mach.site import CommandSiteManager + from mozboot.util import get_state_dir if self._virtualenv_manager is None: - self._virtualenv_manager = CommandSiteManager( + self._virtualenv_manager = CommandSiteManager.from_environment( self.topsrcdir, - os.path.join(self.topobjdir, "_virtualenvs"), + get_state_dir(), self._virtualenv_name, + os.path.join(self.topobjdir, "_virtualenvs"), ) return self._virtualenv_manager diff --git a/python/mozbuild/mozbuild/controller/building.py b/python/mozbuild/mozbuild/controller/building.py index 8748752b8f1d..a60b2893e80d 100644 --- a/python/mozbuild/mozbuild/controller/building.py +++ b/python/mozbuild/mozbuild/controller/building.py @@ -23,13 +23,14 @@ from collections import ( ) from textwrap import TextWrapper +from mach.site import CommandSiteManager + try: import psutil except Exception: psutil = None from mach.mixin.logging import LoggingMixin -from mozboot.util import get_mach_virtualenv_binary import mozfile from mozsystemmonitor.resourcemonitor import SystemResourceMonitor from mozterm.widgets import Footer @@ -1508,15 +1509,15 @@ class BuildDriver(MozbuildObject): if eq == "=": append_env[k] = v - if six.PY3: - python = sys.executable - else: - # Try to get the mach virtualenv Python if we can. - python = get_mach_virtualenv_binary() - if not os.path.exists(python): - python = "python3" + build_site = CommandSiteManager.from_environment( + self.topsrcdir, + self.statedir, + "build", + os.path.join(self.topobjdir, "_virtualenvs"), + ) + build_site.ensure() - command = [python, os.path.join(self.topsrcdir, "configure.py")] + command = [build_site.python_path, os.path.join(self.topsrcdir, "configure.py")] if options: command.extend(options) diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index 2666e6190489..02606ca39b9f 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -2463,7 +2463,6 @@ def create_mach_environment(command_context, force=False): manager = MachSiteManager.from_environment( command_context.topsrcdir, get_state_dir(), - is_mach_create_mach_env_command=True, ) if manager.ensure(force=force): diff --git a/python/mozbuild/mozbuild/sphinx.py b/python/mozbuild/mozbuild/sphinx.py index 8e5379b65cf3..d8710d0d7125 100644 --- a/python/mozbuild/mozbuild/sphinx.py +++ b/python/mozbuild/mozbuild/sphinx.py @@ -198,6 +198,7 @@ class MozbuildSymbols(Directive): def setup(app): from mach.site import CommandSiteManager + from mozboot.util import get_state_dir from moztreedocs import manager app.add_directive("mozbuildsymbols", MozbuildSymbols) @@ -214,9 +215,10 @@ def setup(app): # We need to adjust sys.path in order for Python API docs to get generated # properly. We leverage the in-tree virtualenv for this. topsrcdir = manager.topsrcdir - site = CommandSiteManager( + site = CommandSiteManager.from_environment( topsrcdir, - os.path.join(app.outdir, "_venv"), + get_state_dir(), "common", + os.path.join(app.outdir, "_venv"), ) site.activate() diff --git a/python/mozbuild/mozbuild/test/test_vendor.py b/python/mozbuild/mozbuild/test/test_vendor.py index 3203e880ce02..6e1fcc5b1fc9 100644 --- a/python/mozbuild/mozbuild/test/test_vendor.py +++ b/python/mozbuild/mozbuild/test/test_vendor.py @@ -17,6 +17,17 @@ import mozunit def test_up_to_date_vendor(): with tempfile.TemporaryDirectory() as work_dir: + + def copy_to_work_dir(relative_path): + args = ( + os.path.join(topsrcdir, relative_path), + os.path.join(work_dir, relative_path), + ) + + shutil.copytree(*args) if os.path.isdir(relative_path) else shutil.copy( + *args + ) + subprocess.check_call(["hg", "init", work_dir]) os.makedirs(os.path.join(work_dir, "build")) os.makedirs(os.path.join(work_dir, "third_party")) @@ -34,17 +45,10 @@ def test_up_to_date_vendor(): file.write("vendored:third_party/python/setuptools\n") file.write("vendored:third_party/python/wheel\n") - # Copy existing "third_party/python/" vendored files - existing_vendored = os.path.join(topsrcdir, "third_party", "python") - work_vendored = os.path.join(work_dir, "third_party", "python") - shutil.copytree(existing_vendored, work_vendored) - - # Copy "mach" module so that `CommandSiteManager` can populate itself. - # This is needed because "topsrcdir" is used in this test both for determining - # import paths and for acting as a "work dir". - existing_mach = os.path.join(topsrcdir, "python", "mach") - work_mach = os.path.join(work_dir, "python", "mach") - shutil.copytree(existing_mach, work_mach) + copy_to_work_dir(os.path.join("third_party", "python")) + copy_to_work_dir(os.path.join("python", "mach")) + copy_to_work_dir(os.path.join("build", "mach_virtualenv_packages.txt")) + copy_to_work_dir(os.path.join("build", "common_virtualenv_packages.txt")) # Run the vendoring process vendor = VendorPython( @@ -59,8 +63,8 @@ def test_up_to_date_vendor(): [ "diff", "-r", - existing_vendored, - work_vendored, + os.path.join(topsrcdir, os.path.join("third_party", "python")), + os.path.join(work_dir, os.path.join("third_party", "python")), "--exclude=__pycache__", ] )