Bug 844509 - Don't encode the configuration that configure passes to config_status. r=nalexander

With bug 1575135, we ensure nothing gets out of the configure sandbox
as a bytes string. We can thus now avoid the encode() pass in
configure.py. We still need, however, to normalize the configuration
so that it doesn't contain unexpected types, and conformning to what
indented_repr does to the configuration in config.status.

While here, convert some obj.iteritems() to six.iteritems(obj).
And remove the now unused encode function.

Differential Revision: https://phabricator.services.mozilla.com/D42630

configure.py

@@ -24,7 +24,6 @@ from mozbuild.pythonutil import iter_modules_in_path
 from mozbuild.backend.configenvironment import PartialConfigEnvironment
 from mozbuild.util import (
     indented_repr,
-    encode,
     system_encoding,
 )
 import mozpack.path as mozpath
@@ -82,12 +81,12 @@ def config_status(config):
 
     sanitized_config = {}
     sanitized_config['substs'] = {
-        k: sanitized_bools(v) for k, v in config.iteritems()
+        k: sanitized_bools(v) for k, v in six.iteritems(config)
         if k not in ('DEFINES', 'non_global_defines', 'TOPSRCDIR', 'TOPOBJDIR',
                      'CONFIG_STATUS_DEPS')
     }
     sanitized_config['defines'] = {
-        k: sanitized_bools(v) for k, v in config['DEFINES'].iteritems()
+        k: sanitized_bools(v) for k, v in six.iteritems(config['DEFINES'])
     }
     sanitized_config['non_global_defines'] = config['non_global_defines']
     sanitized_config['topsrcdir'] = config['TOPSRCDIR']
@@ -108,13 +107,9 @@ def config_status(config):
             #!%(python)s
             # coding=%(encoding)s
             from __future__ import unicode_literals
-            from mozbuild.util import encode
-            encoding = '%(encoding)s'
         ''') % {'python': config['PYTHON'], 'encoding': system_encoding})
-        # A lot of the build backend code is currently expecting byte
-        # strings and breaks in subtle ways with unicode strings. (bug 1296508)
-        for k, v in sanitized_config.iteritems():
-            fh.write('%s = encode(%s, encoding)\n' % (k, indented_repr(v)))
+        for k, v in six.iteritems(sanitized_config):
+            fh.write('%s = %s\n' % (k, indented_repr(v)))
         fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', "
                  "'non_global_defines', 'substs', 'mozconfig']")
 
@@ -148,12 +143,21 @@ def config_status(config):
         # Some values in sanitized_config also have more complex types, such as
         # EnumString, which using when calling config_status would currently
         # break the build, as well as making it inconsistent with re-running
-        # config.status. Fortunately, EnumString derives from unicode, so it's
-        # covered by converting unicode strings.
-
-        # A lot of the build backend code is currently expecting byte strings
-        # and breaks in subtle ways with unicode strings.
-        return config_status(args=[], **encode(sanitized_config, system_encoding))
+        # config.status, for which they are normalized to plain strings via
+        # indented_repr. Likewise for non-dict non-string iterables being
+        # converted to lists.
+        def normalize(obj):
+            if isinstance(obj, dict):
+                return {
+                    k: normalize(v)
+                    for k, v in six.iteritems(obj)
+                }
+            if isinstance(obj, six.text_type):
+                return six.text_type(obj)
+            if isinstance(obj, Iterable):
+                return [normalize(o) for o in obj]
+            return obj
+        return config_status(args=[], **normalize(sanitized_config))
     return 0
 
 

python/mozbuild/mozbuild/util.py

@@ -23,7 +23,6 @@ import sys
 import time
 
 from collections import (
-    Iterable,
     OrderedDict,
 )
 from io import (
@@ -1297,22 +1296,6 @@ def indented_repr(o, indent=4):
     return ''.join(recurse_indented_repr(o, 0))
 
 
-def encode(obj, encoding='utf-8'):
-    '''Recursively encode unicode strings with the given encoding.'''
-    if isinstance(obj, dict):
-        return {
-            encode(k, encoding): encode(v, encoding)
-            for k, v in six.iteritems(obj)
-        }
-    if isinstance(obj, bytes):
-        return obj
-    if isinstance(obj, six.text_type):
-        return obj.encode(encoding)
-    if isinstance(obj, Iterable):
-        return [encode(i, encoding) for i in obj]
-    return obj
-
-
 def patch_main():
     '''This is a hack to work around the fact that Windows multiprocessing needs
     to import the original main module, and assumes that it corresponds to a file