Bug 1402012 - Create config.statusd directory; r=glandium
The config.statusd directory is created alongside config.status, which contains the same information but is split across many files instead of all in a single file. This allows the build system to track dependencies on individual configure values. MozReview-Commit-ID: 2DbwKCJuNSX
This commit is contained in:
Mike Shal
@@ -17,6 +17,7 @@ sys.path.insert(0, os.path.join(base_dir, 'python', 'mozbuild'))
|
||||
from mozbuild.configure import ConfigureSandbox
|
||||
from mozbuild.makeutil import Makefile
|
||||
from mozbuild.pythonutil import iter_modules_in_path
|
||||
from mozbuild.backend.configenvironment import PartialConfigEnvironment
|
||||
from mozbuild.util import (
|
||||
indented_repr,
|
||||
encode,
|
||||
@@ -90,6 +91,9 @@ def config_status(config):
|
||||
config_status(**args)
|
||||
'''))
|
||||
|
||||
partial_config = PartialConfigEnvironment(config['TOPOBJDIR'])
|
||||
partial_config.write_vars(sanitized_config)
|
||||
|
||||
# Write out a depfile so Make knows to re-run configure when relevant Python
|
||||
# changes.
|
||||
mk = Makefile()
|
||||
|
||||
@@ -6,13 +6,15 @@ from __future__ import absolute_import
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
|
||||
from collections import Iterable
|
||||
from collections import Iterable, OrderedDict
|
||||
from types import StringTypes, ModuleType
|
||||
|
||||
import mozpack.path as mozpath
|
||||
|
||||
from mozbuild.util import (
|
||||
FileAvoidWrite,
|
||||
memoized_property,
|
||||
ReadOnlyDict,
|
||||
)
|
||||
@@ -211,3 +213,154 @@ class ConfigEnvironment(object):
|
||||
|
||||
return ConfigEnvironment(config.topsrcdir, config.topobjdir,
|
||||
config.defines, config.non_global_defines, config.substs, path)
|
||||
|
||||
|
||||
class PartialConfigDict(object):
|
||||
"""Facilitates mapping the config.statusd defines & substs with dict-like access.
|
||||
|
||||
This allows a buildconfig client to use buildconfig.defines['FOO'] (and
|
||||
similar for substs), where the value of FOO is delay-loaded until it is
|
||||
needed.
|
||||
"""
|
||||
def __init__(self, config_statusd, typ, environ_override=False):
|
||||
self._dict = {}
|
||||
self._datadir = mozpath.join(config_statusd, typ)
|
||||
self._config_track = mozpath.join(self._datadir, 'config.track')
|
||||
self._files = set()
|
||||
self._environ_override = environ_override
|
||||
|
||||
def _load_config_track(self):
|
||||
existing_files = set()
|
||||
try:
|
||||
with open(self._config_track) as fh:
|
||||
existing_files.update(fh.read().splitlines())
|
||||
except IOError:
|
||||
pass
|
||||
return existing_files
|
||||
|
||||
def _write_file(self, key, value):
|
||||
filename = mozpath.join(self._datadir, key)
|
||||
with FileAvoidWrite(filename) as fh:
|
||||
json.dump(value, fh, indent=4)
|
||||
return filename
|
||||
|
||||
def _fill_group(self, values):
|
||||
# Clear out any cached values. This is mostly for tests that will check
|
||||
# the environment, write out a new set of variables, and then check the
|
||||
# environment again. Normally only configure ends up calling this
|
||||
# function, and other consumers create their own
|
||||
# PartialConfigEnvironments in new python processes.
|
||||
self._dict = {}
|
||||
|
||||
existing_files = self._load_config_track()
|
||||
|
||||
new_files = set()
|
||||
for k, v in values.iteritems():
|
||||
new_files.add(self._write_file(k, v))
|
||||
|
||||
for filename in existing_files - new_files:
|
||||
# We can't actually os.remove() here, since make would not see that the
|
||||
# file has been removed and that the target needs to be updated. Instead
|
||||
# we just overwrite the file with a value of None, which is equivalent
|
||||
# to a non-existing file.
|
||||
with FileAvoidWrite(filename) as fh:
|
||||
json.dump(None, fh)
|
||||
|
||||
with FileAvoidWrite(self._config_track) as fh:
|
||||
for f in sorted(new_files):
|
||||
fh.write('%s\n' % f)
|
||||
|
||||
def __getitem__(self, key):
|
||||
if self._environ_override:
|
||||
if (key not in ('CPP', 'CXXCPP', 'SHELL')) and (key in os.environ):
|
||||
return os.environ[key]
|
||||
|
||||
if key not in self._dict:
|
||||
data = None
|
||||
try:
|
||||
filename = mozpath.join(self._datadir, key)
|
||||
self._files.add(filename)
|
||||
with open(filename) as f:
|
||||
data = json.load(f)
|
||||
except IOError:
|
||||
pass
|
||||
self._dict[key] = data
|
||||
|
||||
if self._dict[key] is None:
|
||||
raise KeyError("'%s'" % key)
|
||||
return self._dict[key]
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
self._dict[key] = value
|
||||
|
||||
def get(self, key, default=None):
|
||||
return self[key] if key in self else default
|
||||
|
||||
def __contains__(self, key):
|
||||
try:
|
||||
return self[key] is not None
|
||||
except KeyError:
|
||||
return False
|
||||
|
||||
def iteritems(self):
|
||||
existing_files = self._load_config_track()
|
||||
for f in existing_files:
|
||||
# The track file contains filenames, and the basename is the
|
||||
# variable name.
|
||||
var = mozpath.basename(f)
|
||||
yield var, self[var]
|
||||
|
||||
|
||||
class PartialConfigEnvironment(object):
|
||||
"""Allows access to individual config.status items via config.statusd/* files.
|
||||
|
||||
This class is similar to the full ConfigEnvironment, which uses
|
||||
config.status, except this allows access and tracks dependencies to
|
||||
individual configure values. It is intended to be used during the build
|
||||
process to handle things like GENERATED_FILES, CONFIGURE_DEFINE_FILES, and
|
||||
anything else that may need to access specific substs or defines.
|
||||
|
||||
Creating a PartialConfigEnvironment requires only the topobjdir, which is
|
||||
needed to distinguish between the top-level environment and the js/src
|
||||
environment.
|
||||
|
||||
The PartialConfigEnvironment automatically defines one additional subst variable
|
||||
from all the defines not appearing in non_global_defines:
|
||||
- ACDEFINES contains the defines in the form -DNAME=VALUE, for use on
|
||||
preprocessor command lines. The order in which defines were given
|
||||
when creating the ConfigEnvironment is preserved.
|
||||
|
||||
and one additional define from all the defines as a dictionary:
|
||||
- ALLDEFINES contains all of the global defines as a dictionary. This is
|
||||
intended to be used instead of the defines structure from config.status so
|
||||
that scripts can depend directly on its value.
|
||||
"""
|
||||
def __init__(self, topobjdir):
|
||||
config_statusd = mozpath.join(topobjdir, 'config.statusd')
|
||||
self.substs = PartialConfigDict(config_statusd, 'substs', environ_override=True)
|
||||
self.defines = PartialConfigDict(config_statusd, 'defines')
|
||||
self.topobjdir = topobjdir
|
||||
|
||||
def write_vars(self, config):
|
||||
substs = config['substs'].copy()
|
||||
defines = config['defines'].copy()
|
||||
|
||||
global_defines = [
|
||||
name for name in config['defines']
|
||||
if name not in config['non_global_defines']
|
||||
]
|
||||
acdefines = ' '.join(['-D%s=%s' % (name,
|
||||
shell_quote(config['defines'][name]).replace('$', '$$'))
|
||||
for name in sorted(global_defines)])
|
||||
substs['ACDEFINES'] = acdefines
|
||||
|
||||
all_defines = OrderedDict()
|
||||
for k in global_defines:
|
||||
all_defines[k] = config['defines'][k]
|
||||
defines['ALLDEFINES'] = all_defines
|
||||
|
||||
self.substs._fill_group(substs)
|
||||
self.defines._fill_group(defines)
|
||||
|
||||
def get_dependencies(self):
|
||||
return ['$(wildcard %s)' % f for f in self.substs._files | self.defines._files]
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from mozunit import main
|
||||
from tempfile import mkdtemp
|
||||
from shutil import rmtree
|
||||
|
||||
import mozpack.path as mozpath
|
||||
from mozbuild.backend.configenvironment import PartialConfigEnvironment
|
||||
|
||||
config = {
|
||||
'defines': {
|
||||
'MOZ_FOO': '1',
|
||||
'MOZ_BAR': '2',
|
||||
'MOZ_NON_GLOBAL': '3',
|
||||
},
|
||||
'substs': {
|
||||
'MOZ_SUBST_1': '1',
|
||||
'MOZ_SUBST_2': '2',
|
||||
'CPP': 'cpp',
|
||||
},
|
||||
'non_global_defines': [
|
||||
'MOZ_NON_GLOBAL',
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class TestPartial(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._old_env = dict(os.environ)
|
||||
|
||||
def tearDown(self):
|
||||
os.environ.clear()
|
||||
os.environ.update(self._old_env)
|
||||
|
||||
def _objdir(self):
|
||||
objdir = mkdtemp()
|
||||
self.addCleanup(rmtree, objdir)
|
||||
return objdir
|
||||
|
||||
def test_auto_substs(self):
|
||||
'''Test the automatically set values of ACDEFINES, and ALLDEFINES
|
||||
'''
|
||||
env = PartialConfigEnvironment(self._objdir())
|
||||
env.write_vars(config)
|
||||
self.assertEqual(env.substs['ACDEFINES'], '-DMOZ_BAR=2 -DMOZ_FOO=1')
|
||||
self.assertEqual(env.defines['ALLDEFINES'], {
|
||||
'MOZ_BAR': '2',
|
||||
'MOZ_FOO': '1',
|
||||
})
|
||||
|
||||
def test_remove_subst(self):
|
||||
'''Test removing a subst from the config. The file should be overwritten with 'None'
|
||||
'''
|
||||
env = PartialConfigEnvironment(self._objdir())
|
||||
path = mozpath.join(env.topobjdir, 'config.statusd', 'substs', 'MYSUBST')
|
||||
myconfig = config.copy()
|
||||
env.write_vars(myconfig)
|
||||
with self.assertRaises(KeyError):
|
||||
x = env.substs['MYSUBST']
|
||||
self.assertFalse(os.path.exists(path))
|
||||
|
||||
myconfig['substs']['MYSUBST'] = 'new'
|
||||
env.write_vars(myconfig)
|
||||
|
||||
self.assertEqual(env.substs['MYSUBST'], 'new')
|
||||
self.assertTrue(os.path.exists(path))
|
||||
|
||||
del myconfig['substs']['MYSUBST']
|
||||
env.write_vars(myconfig)
|
||||
with self.assertRaises(KeyError):
|
||||
x = env.substs['MYSUBST']
|
||||
# Now that the subst is gone, the file still needs to be present so that
|
||||
# make can update dependencies correctly. Overwriting the file with
|
||||
# 'None' is the same as deleting it as far as the
|
||||
# PartialConfigEnvironment is concerned, but make can't track a
|
||||
# dependency on a file that doesn't exist.
|
||||
self.assertTrue(os.path.exists(path))
|
||||
|
||||
def _assert_deps(self, env, deps):
|
||||
deps = sorted(['$(wildcard %s)' % (mozpath.join(env.topobjdir, 'config.statusd', d)) for d in deps])
|
||||
self.assertEqual(sorted(env.get_dependencies()), deps)
|
||||
|
||||
def test_dependencies(self):
|
||||
'''Test getting dependencies on defines and substs.
|
||||
'''
|
||||
env = PartialConfigEnvironment(self._objdir())
|
||||
env.write_vars(config)
|
||||
self._assert_deps(env, [])
|
||||
|
||||
self.assertEqual(env.defines['MOZ_FOO'], '1')
|
||||
self._assert_deps(env, ['defines/MOZ_FOO'])
|
||||
|
||||
self.assertEqual(env.defines['MOZ_BAR'], '2')
|
||||
self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR'])
|
||||
|
||||
# Getting a define again shouldn't add a redundant dependency
|
||||
self.assertEqual(env.defines['MOZ_FOO'], '1')
|
||||
self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR'])
|
||||
|
||||
self.assertEqual(env.substs['MOZ_SUBST_1'], '1')
|
||||
self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR', 'substs/MOZ_SUBST_1'])
|
||||
|
||||
with self.assertRaises(KeyError):
|
||||
x = env.substs['NON_EXISTENT']
|
||||
self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR', 'substs/MOZ_SUBST_1', 'substs/NON_EXISTENT'])
|
||||
self.assertEqual(env.substs.get('NON_EXISTENT'), None)
|
||||
|
||||
def test_set_subst(self):
|
||||
'''Test setting a subst
|
||||
'''
|
||||
env = PartialConfigEnvironment(self._objdir())
|
||||
env.write_vars(config)
|
||||
|
||||
self.assertEqual(env.substs['MOZ_SUBST_1'], '1')
|
||||
env.substs['MOZ_SUBST_1'] = 'updated'
|
||||
self.assertEqual(env.substs['MOZ_SUBST_1'], 'updated')
|
||||
|
||||
# A new environment should pull the result from the file again.
|
||||
newenv = PartialConfigEnvironment(env.topobjdir)
|
||||
self.assertEqual(newenv.substs['MOZ_SUBST_1'], '1')
|
||||
|
||||
def test_env_override(self):
|
||||
'''Test overriding a subst with an environment variable
|
||||
'''
|
||||
env = PartialConfigEnvironment(self._objdir())
|
||||
env.write_vars(config)
|
||||
|
||||
self.assertEqual(env.substs['MOZ_SUBST_1'], '1')
|
||||
self.assertEqual(env.substs['CPP'], 'cpp')
|
||||
|
||||
# Reset the environment and set some environment variables.
|
||||
env = PartialConfigEnvironment(env.topobjdir)
|
||||
os.environ['MOZ_SUBST_1'] = 'subst 1 environ'
|
||||
os.environ['CPP'] = 'cpp environ'
|
||||
|
||||
# The MOZ_SUBST_1 should be overridden by the environment, while CPP is
|
||||
# a special variable and should not.
|
||||
self.assertEqual(env.substs['MOZ_SUBST_1'], 'subst 1 environ')
|
||||
self.assertEqual(env.substs['CPP'], 'cpp')
|
||||
|
||||
def test_update(self):
|
||||
'''Test calling update on the substs or defines pseudo dicts
|
||||
'''
|
||||
env = PartialConfigEnvironment(self._objdir())
|
||||
env.write_vars(config)
|
||||
|
||||
mysubsts = {'NEW': 'new'}
|
||||
mysubsts.update(env.substs.iteritems())
|
||||
self.assertEqual(mysubsts['NEW'], 'new')
|
||||
self.assertEqual(mysubsts['CPP'], 'cpp')
|
||||
|
||||
mydefines = {'DEBUG': '1'}
|
||||
mydefines.update(env.defines.iteritems())
|
||||
self.assertEqual(mydefines['DEBUG'], '1')
|
||||
self.assertEqual(mydefines['MOZ_FOO'], '1')
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -3,6 +3,7 @@
|
||||
[action/test_package_fennec_apk.py]
|
||||
[backend/test_build.py]
|
||||
[backend/test_configenvironment.py]
|
||||
[backend/test_partialconfigenvironment.py]
|
||||
[backend/test_recursivemake.py]
|
||||
[backend/test_test_manifest.py]
|
||||
[backend/test_visualstudio.py]
|
||||
|
||||
Reference in New Issue
Block a user