Bug 1943612 - Provide generic machinery to load confvars.sh as a simple key/value format r=glandium

Doing so prevents users from putting customization à la moz.configure in
the confvars.sh file, which should enforce better practices.

Share the implementation with repackaging/msix.py and test it.

Differential Revision: https://phabricator.services.mozilla.com/D235769

build/moz.configure/util.configure

@@ -540,6 +540,46 @@ def project_flag(env=None, set_as_define=False, **kwargs):
         set_define(env, option_implementation)
 
 
+# A template providing a shorthand for setting a variable. The created
+# option will only be settable from a confvars.sh file.
+# If required, the set_as_define argument will additionally cause the variable
+# to be set using set_define.
+# Similarly, set_as_config can be set to False if the variable should not be
+# passed to set_config.
+@template
+def confvar(
+    env=None,
+    set_as_config=True,
+    set_as_define=False,
+    allow_implied=False,
+    **kwargs,
+):
+    if not env:
+        configure_error("A project_flag must be passed a variable name to set.")
+
+    if kwargs.get("nargs", 0) not in (0, 1):
+        configure_error("A project_flag must be passed nargs={0,1}.")
+
+    origins = ("confvars",)
+    if allow_implied:
+        origins += ("implied",)
+    opt = option(env=env, possible_origins=origins, **kwargs)
+
+    @depends(opt.option)
+    def option_implementation(value):
+        if value:
+            if len(value) == 1:
+                return value[0]
+            elif len(value):
+                return value
+            return bool(value)
+
+    if set_as_config:
+        set_config(env, option_implementation)
+    if set_as_define:
+        set_define(env, option_implementation)
+
+
 @template
 @imports(_from="mozbuild.configure.constants", _import="RaiseErrorOnUse")
 def obsolete_config(name, *, replacement):

moz.configure

@@ -440,6 +440,29 @@ pack_relative_relocs_flags = dependable(False)
 include(include_project_configure)
 
 
+@depends(build_environment, build_project, "--help")
+@checking("if project-specific confvars.sh exists")
+# This gives access to the sandbox. Don't copy this blindly.
+@imports("__sandbox__")
+@imports(_from="mozbuild.configure", _import="confvars")
+@imports("os.path")
+def load_confvars(build_env, build_project, help):
+    confvars_path = os.path.join(build_env.topsrcdir, build_project, "confvars.sh")
+    if os.path.exists(confvars_path):
+        helper = __sandbox__._helper
+        # parse confvars
+        try:
+            keyvals = confvars.parse(confvars_path)
+        except confvars.ConfVarsSyntaxError as e:
+            die(str(e))
+        for key, value in keyvals.items():
+            # FIXME: remove test once we no longer load confvars from old-configure.
+            if key in __sandbox__._options:
+                # ~= imply_option, but with an accurate origin
+                helper.add(f"{key}={value}", origin="confvars", args=helper._args)
+        return confvars_path
+
+
 # Final flags validation and gathering
 # -------------------------------------------------
 

python/mozbuild/mozbuild/configure/confvars.py

@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozbuild.shellutil
+
+
+class ConfVarsSyntaxError(SyntaxError):
+    def __init__(self, msg, file, lineno, colnum, line):
+        super().__init__(msg, (file, lineno, colnum, line))
+
+
+def parse(path):
+    with open(path) as confvars:
+        keyvals = {}
+        for lineno, rawline in enumerate(confvars, start=1):
+            line = rawline.rstrip()
+            # Empty line / comment.
+            line_no_leading_blank = line.lstrip()
+            if not line_no_leading_blank or line_no_leading_blank.startswith("#"):
+                continue
+
+            head, sym, tail = line.partition("=")
+            if sym != "=" or "#" in head:
+                raise ConfVarsSyntaxError(
+                    "Expecting key=value format", path, lineno, 1, line
+                )
+            key = head.strip()
+
+            # Verify there's no unexpected spaces.
+            if key != head:
+                colno = 1 + line.index(key)
+                raise ConfVarsSyntaxError(
+                    f"Expecting no spaces around '{key}'", path, lineno, colno, line
+                )
+            if tail.lstrip() != tail:
+                colno = 1 + line.index(tail)
+                raise ConfVarsSyntaxError(
+                    f"Expecting no spaces between '=' and '{tail.lstrip()}'",
+                    path,
+                    lineno,
+                    colno,
+                    line,
+                )
+
+            # Verify we don't have duplicate keys.
+            if key in keyvals:
+                raise ConfVarsSyntaxError(
+                    f"Invalid redefinition for '{key}'",
+                    path,
+                    lineno,
+                    1 + line.index(key),
+                    line,
+                )
+
+            # Parse value.
+            try:
+                values = mozbuild.shellutil.split(tail)
+            except mozbuild.shellutil.MetaCharacterException as e:
+                raise ConfVarsSyntaxError(
+                    f"Unquoted, non-escaped special character '{e.char}'",
+                    path,
+                    lineno,
+                    1 + line.index(e.char),
+                    line,
+                )
+            except Exception as e:
+                raise ConfVarsSyntaxError(
+                    e.args[0].replace(" in command", ""),
+                    path,
+                    lineno,
+                    1 + line.index("="),
+                    line,
+                )
+            value = values[0] if values else ""
+
+            # Finally, commit the key<> value pair \o/.
+            keyvals[key] = value
+        return keyvals

python/mozbuild/mozbuild/repackaging/msix.py

@@ -33,6 +33,7 @@ from mozpack.mozjar import JarReader
 from mozpack.packager.unpack import UnpackFinder
 from six.moves import shlex_quote
 
+from mozbuild.configure import confvars
 from mozbuild.dirutils import ensureParentDir
 from mozbuild.repackaging.application_ini import get_application_ini_values
 
@@ -208,29 +209,23 @@ def get_appconstants_sys_mjs_values(finder, *args):
 
 def get_branding(use_official, topsrcdir, build_app, finder, log=None):
     """Figure out which branding directory to use."""
-    conf_vars = mozpath.join(topsrcdir, build_app, "confvars.sh")
+    confvars_path = mozpath.join(topsrcdir, build_app, "confvars.sh")
-
+    confvars_content = confvars.parse(confvars_path)
-    def conf_vars_value(key):
+    for key, value in confvars_content.items():
-        lines = [line.strip() for line in open(conf_vars).readlines()]
-        for line in lines:
-            if line and line[0] == "#":
-                continue
-            if key not in line:
-                continue
-            _, _, value = line.partition("=")
-            if not value:
-                continue
         log(
             logging.INFO,
             "msix",
-                {"key": key, "conf_vars": conf_vars, "value": value},
+            {"key": key, "conf_vars": confvars_path, "value": value},
             "Read '{key}' from {conf_vars}: {value}",
         )
-            return value
+
+    def conf_vars_value(key):
+        if key in confvars_content:
+            return confvars_content[key]
         log(
             logging.ERROR,
             "msix",
-            {"key": key, "conf_vars": conf_vars},
+            {"key": key, "conf_vars": confvars_content},
             "Unable to find '{key}' in {conf_vars}!",
         )
 

python/mozbuild/mozbuild/test/configure/data/confvars/confvars.sh

@@ -0,0 +1,4 @@
+# line comment
+CONFVAR=" a b c"
+OTHER_CONFVAR=d # trailing comment
+

python/mozbuild/mozbuild/test/configure/data/confvars/moz.configure

@@ -0,0 +1,10 @@
+confvar(
+    "CONFVAR",
+    nargs=1,
+    help="Confvar",
+)
+confvar(
+    "OTHER_CONFVAR",
+    nargs=1,
+    help="Other confvar",
+)

python/mozbuild/mozbuild/test/configure/test_moz_configure.py

@@ -193,5 +193,20 @@ class TestMozConfigure(BaseConfigureTest):
         self.assertEqual(check_nsis_version("v3.1"), "3.1")
 
 
+class TestConfVars(BaseConfigureTest):
+    def test_loading(self):
+        sandbox = self.get_sandbox(
+            paths={},
+            config={},
+            args=[
+                "--enable-project=python/mozbuild/mozbuild/test/configure/data/confvars"
+            ],
+        )
+        self.assertEqual(
+            list(sandbox._helper),
+            ["CONFVAR= a b c", "OTHER_CONFVAR=d"],
+        )
+
+
 if __name__ == "__main__":
     main()

python/mozbuild/mozbuild/test/python.toml

@@ -79,6 +79,8 @@ subsuite = "mozbuild"
 
 ["test_base.py"]
 
+["test_confvars.py"]
+
 ["test_containers.py"]
 
 ["test_dotproperties.py"]

python/mozbuild/mozbuild/test/test_confvars.py

@@ -0,0 +1,139 @@
+import os
+import re
+import unittest
+from tempfile import NamedTemporaryFile
+
+import mozunit
+
+from mozbuild.configure.confvars import ConfVarsSyntaxError, parse
+
+
+def TemporaryConfVars():
+    return NamedTemporaryFile("wt", delete=False)
+
+
+class TestContext(unittest.TestCase):
+
+    def loads(self, *lines):
+        with NamedTemporaryFile("wt", delete=False) as ntf:
+            ntf.writelines(lines)
+        try:
+            confvars = parse(ntf.name)
+        finally:
+            os.remove(ntf.name)
+        return confvars
+
+    def test_parse_empty_file(self):
+        confvars = self.loads("# comment\n")
+        self.assertEqual(confvars, {})
+
+    def test_parse_simple_assignment(self):
+        confvars = self.loads("a=b\n")
+        self.assertEqual(confvars, {"a": "b"})
+
+    def test_parse_simple_assignment_with_equal_in_value(self):
+        confvars = self.loads("a='='\n", "b==")
+        self.assertEqual(confvars, {"a": "=", "b": "="})
+
+    def test_parse_simple_assignment_with_sharp_in_value(self):
+        confvars = self.loads("a='#'\n")
+        self.assertEqual(confvars, {"a": "#"})
+
+    def test_parse_simple_assignment_with_trailing_spaces(self):
+        confvars = self.loads("a1=1\t\n", "\n", "a2=2\n", "a3=3 \n", "a4=4")
+
+        self.assertEqual(
+            confvars,
+            {
+                "a1": "1",
+                "a2": "2",
+                "a3": "3",
+                "a4": "4",
+            },
+        )
+
+    def test_parse_trailing_comment(self):
+        confvars = self.loads("a=b#comment\n")
+        self.assertEqual(confvars, {"a": "b"})
+
+    def test_parse_invalid_assign_in_trailing_comment(self):
+        with self.assertRaises(ConfVarsSyntaxError) as cm:
+            self.loads("a#=comment\n")
+        self.assertTrue(
+            re.match("Expecting key=value format \\(.*, line 1\\)", str(cm.exception))
+        )
+
+    def test_parse_quoted_assignment(self):
+        confvars = self.loads("a='b'\n" "b=' c'\n" 'c=" \'c"\n')
+        self.assertEqual(confvars, {"a": "b", "b": " c", "c": " 'c"})
+
+    def test_parse_invalid_assignment(self):
+        with self.assertRaises(ConfVarsSyntaxError) as cm:
+            self.loads("a#comment\n")
+        self.assertTrue(
+            re.match("Expecting key=value format \\(.*, line 1\\)", str(cm.exception))
+        )
+
+    def test_parse_empty_value(self):
+        confvars = self.loads("a=\n")
+        self.assertEqual(confvars, {"a": ""})
+
+    def test_parse_invalid_value(self):
+        with self.assertRaises(ConfVarsSyntaxError) as cm:
+            self.loads("#comment\na='er\n")
+        self.assertTrue(
+            re.match(
+                "Unterminated quoted string \\(.*, line 2\\)",
+                str(cm.exception),
+            )
+        )
+        with self.assertRaises(ConfVarsSyntaxError) as cm:
+            self.loads("a= er\n")
+        self.assertTrue(
+            re.match(
+                "Expecting no spaces between '=' and 'er' \\(.*, line 1\\)",
+                str(cm.exception),
+            )
+        )
+
+    def test_parse_invalid_char(self):
+        with self.assertRaises(ConfVarsSyntaxError) as cm:
+            self.loads("a=$\n")
+        self.assertTrue(
+            re.match(
+                "Unquoted, non-escaped special character '\\$' \\(.*, line 1\\)",
+                str(cm.exception),
+            )
+        )
+
+    def test_parse_invalid_key(self):
+        with self.assertRaises(ConfVarsSyntaxError) as cm:
+            self.loads(" a=1\n")
+        self.assertTrue(
+            re.match(
+                "Expecting no spaces around 'a' \\(.*, line 1\\)",
+                str(cm.exception),
+            )
+        )
+        with self.assertRaises(ConfVarsSyntaxError) as cm:
+            self.loads("a =1\n")
+        self.assertTrue(
+            re.match(
+                "Expecting no spaces around 'a' \\(.*, line 1\\)",
+                str(cm.exception),
+            )
+        )
+
+    def test_parse_redundant_key(self):
+        with self.assertRaises(ConfVarsSyntaxError) as cm:
+            self.loads("a=1\na=2\n")
+        self.assertTrue(
+            re.match(
+                "Invalid redefinition for 'a' \\(.*, line 2\\)",
+                str(cm.exception),
+            )
+        )
+
+
+if __name__ == "__main__":
+    mozunit.main()

toolkit/moz.configure

@@ -124,7 +124,7 @@ def all_configure_options():
         # defaults.
         if (
             value is not None
-            and value.origin not in ("default", "implied")
+            and value.origin not in ("default", "implied", "confvars")
             and value != option.default
         ):
             result.append(