corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1730712: Use consistent Python version throughout configure r=andi

Browse Source 
It's possible for the `PYTHON3` config to point to a different Python
than that that is executing the configure scripts themselves.

This flexibility was needed for the Python 2->3 migration, but now that
it's complete, we can remove the extra configuration and just lean on
the Python interpreter used to run configure.

Differential Revision: https://phabricator.services.mozilla.com/D129863
This commit is contained in:
Mitchell Hentges
2021-11-04 21:41:33 +00:00
parent 5403fedaff
commit 75f9d0ea8e
3 changed files with 33 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
build/moz.configure/init.configure
View File

@@ -215,165 +215,17 @@ shell = help_shell | shell
# Python 3
# ========
option(env="PYTHON3", nargs=1, help="Python 3 interpreter (3.6 or later)")
@depends("PYTHON3", check_build_environment, mozconfig, "--help")
@imports(_from="__builtin__", _import="Exception")
@imports("os")
@imports("sys")
@imports("subprocess")
@imports("distutils.sysconfig")
@imports(_from="mozbuild.configure.util", _import="LineIO")
@imports(_from="mach.virtualenv", _import="VirtualenvManager")
@imports(_from="mozbuild.pythonutil", _import="find_python3_executable")
@imports(_from="mozbuild.pythonutil", _import="python_executable_version")
@imports(_from="six", _import="ensure_text")
def virtualenv_python3(env_python, build_env, mozconfig, help):
# Avoid re-executing python when running configure --help.
if help:
return
# NOTE: We cannot assume the Python we are calling this code with is the
# Python we want to set up a virtualenv for.
#
# We also cannot assume that the Python the caller is configuring meets our
# build requirements.
#
# Because of this the code is written to re-execute itself with the correct
# interpreter if required.
log.debug("python3: running with pid %r" % os.getpid())
log.debug("python3: sys.executable: %r" % sys.executable)
python = env_python[0] if env_python else None
# Did our python come from mozconfig? Overrides environment setting.
# Ideally we'd rely on the mozconfig injection from mozconfig_options,
# but we'd rather avoid the verbosity when we need to reexecute with
# a different python.
if mozconfig["path"]:
if "PYTHON3" in mozconfig["env"]["added"]:
python = mozconfig["env"]["added"]["PYTHON3"]
elif "PYTHON3" in mozconfig["env"]["modified"]:
python = mozconfig["env"]["modified"]["PYTHON3"][1]
elif "PYTHON3" in mozconfig["vars"]["added"]:
python = mozconfig["vars"]["added"]["PYTHON3"]
elif "PYTHON3" in mozconfig["vars"]["modified"]:
python = mozconfig["vars"]["modified"]["PYTHON3"][1]
log.debug("python3: executable from configuration: %r" % python)
# If this is a mozilla-central build, we'll find the virtualenv in the top
# source directory. If this is a SpiderMonkey build, we assume we're at
# js/src and try to find the virtualenv from the mozilla-central root.
# See mozilla-central changeset d2cce982a7c809815d86d5daecefe2e7a563ecca
# Bug 784841
topsrcdir, topobjdir = build_env.topsrcdir, build_env.topobjdir
if topobjdir.endswith("/js/src"):
topobjdir = topobjdir[:-7]
# Update the path to include some necessary modules for find_program.
sys.path.insert(0, os.path.join(topsrcdir, "testing", "mozbase", "mozfile"))
# If we know the Python executable the caller is asking for then verify its
# version. If the caller did not ask for a specific executable then find
# a reasonable default.
if python:
found_python = find_program(python)
if not found_python:
die(
"The PYTHON3 environment variable does not contain "
"a valid path. Cannot find %s",
python,
)
python = found_python
try:
version = python_executable_version(python).version
except Exception as e:
raise FatalCheckError(
"could not determine version of PYTHON3 " "(%s): %s" % (python, e)
)
else:
# Fall back to the search routine.
python, version = find_python3_executable(min_version="3.6.0")
# The API returns a bytes whereas everything in configure is unicode.
if python:
python = ensure_text(python)
if not python:
raise FatalCheckError(
"Python 3.6 or newer is required to build. "
"Ensure a `python3.x` executable is in your "
"PATH or define PYTHON3 to point to a Python "
"3.6 executable."
)
if version < (3, 6, 0):
raise FatalCheckError(
"Python 3.6 or newer is required to build; "
"%s is Python %d.%d" % (python, version[0], version[1])
)
log.debug("python3: found executable: %r" % python)
test_python = os.environ.get("MOZ_TEST_PYTHON")
if test_python:
manager = namespace(
up_to_date=lambda: True,
python_path=test_python,
)
else:
manager = VirtualenvManager(
topsrcdir,
os.path.join(topobjdir, "_virtualenvs"),
"build",
base_python=python,
)
if not manager.up_to_date():
log.info("Creating Python 3 environment")
manager.build()
else:
log.debug("python3: venv is up to date")
python = normsep(manager.python_path)
if python != normsep(sys.executable):
log.debug(
"python3: executing as %s, should be running as %s"
% (sys.executable, manager.python_path)
)
log.info("Re-executing in the virtualenv")
if env_python:
del os.environ["PYTHON3"]
# Another quirk on macOS, with the system python, the virtualenv is
# not fully operational (missing entries in sys.path) if
# __PYVENV_LAUNCHER__ is set.
os.environ.pop("__PYVENV_LAUNCHER__", None)
# One would prefer to use os.execl, but that's completely borked on
# Windows.
sys.exit(subprocess.call([python] + sys.argv))
# We are now in the virtualenv
if not distutils.sysconfig.get_python_lib():
die("Could not determine python site packages directory")
str_version = ".".join(str(v) for v in version)
return namespace(
path=python,
version=version,
str_version=str_version,
)
@depends(virtualenv_python3)
@dependable
@checking("for Python 3", callback=lambda x: "%s (%s)" % (x.path, x.str_version))
def virtualenv_python3(venv):
return venv
@imports("sys")
@imports(_from="mach.virtualenv", _import="VirtualenvHelper")
def virtualenv_python3():
return namespace(
# sys.executable is currently not updated for in-process activations. However,
# sys.prefix is, so we can calculate the python executable's path from there.
path=normsep(VirtualenvHelper(sys.prefix).python_path),
str_version=".".join(str(i) for i in sys.version_info[0:3]),
)
set_config("PYTHON3", virtualenv_python3.path)

23
configure.py
View File

@@ -22,6 +22,7 @@ sys.path.insert(0, os.path.join(base_dir, "python", "mozbuild"))
sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "packaging"))
sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "pyparsing"))
sys.path.insert(0, os.path.join(base_dir, "third_party", "python", "six"))
from mach.virtualenv import VirtualenvManager
from mozbuild.configure import (
ConfigureSandbox,
TRACE,
@@ -34,6 +35,7 @@ import six
def main(argv):
_activate_build_virtualenv()
config = {}
if "OLD_CONFIGURE" not in os.environ:
@@ -223,5 +225,26 @@ def config_status(config, execute=True):
return 0
def _activate_build_virtualenv():
version = ".".join(str(i) for i in sys.version_info[0:3])
print(f"Using Python {version} from {sys.executable}")
topobjdir = os.path.realpath(".")
topsrcdir = os.path.realpath(os.path.dirname(__file__))
if topobjdir.endswith("/js/src"):
topobjdir = topobjdir[:-7]
build_venv = VirtualenvManager(
topsrcdir,
os.path.join(topobjdir, "_virtualenvs"),
"build",
)
if not build_venv.up_to_date():
print("Creating Python 3 virtualenv")
build_venv.build()
build_venv.activate()
if __name__ == "__main__":
sys.exit(main(sys.argv))

1
python/mozbuild/mozbuild/test/configure/common.py
View File

@@ -292,7 +292,6 @@ class BaseConfigureTest(unittest.TestCase):
environ,
OLD_CONFIGURE=os.path.join(topsrcdir, "old-configure"),
MOZCONFIG=mozconfig_path,
MOZ_TEST_PYTHON=sys.executable,
)
paths = dict(paths)