Bug 1389099 - Properly encode multi-byte translations in CSS r=kmag

The localization filter was not unicode-aware because convertToStream
assigns the output to a nsIStringInputStream (which takes 8-bit chars).
The input was read as a 8-bit string, but after localization it can
contain wide strings if a translation has a multi-byte character.

To fix this, the input stream is now first read as a UTF-8 string, then
localized, and finally exported via a nsIArrayBufferInputStream..

MozReview-Commit-ID: LjCxczIFKCR

toolkit/components/extensions/test/xpcshell/test_ext_i18n_css.js

@@ -20,6 +20,10 @@ AddonTestUtils.init(this);
 
 createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "42");
 
+// Some multibyte characters. This sample was taken from the encoding/api-basics.html web platform test.
+const MULTIBYTE_STRING = "z\xA2\u6C34\uD834\uDD1E\uF8FF\uDBFF\uDFFD\uFFFE";
+let getCSS = (a, b) => `a { content: '${a}'; } b { content: '${b}'; }`;
+
 let extensionData = {
   background: function() {
     function backgroundFetch(url) {
@@ -42,7 +46,7 @@ let extensionData = {
       browser.test.notifyPass("i18n-css");
     });
 
-    browser.test.sendMessage("ready", browser.runtime.getURL("foo.css"));
+    browser.test.sendMessage("ready", browser.runtime.getURL("/"));
   },
 
   manifest: {
@@ -52,7 +56,7 @@ let extensionData = {
       },
     },
 
-    "web_accessible_resources": ["foo.css", "foo.txt", "locale.css"],
+    "web_accessible_resources": ["foo.css", "foo.txt", "locale.css", "multibyte.css"],
 
     "content_scripts": [
       {
@@ -75,6 +79,9 @@ let extensionData = {
         "message": "max-width: 42px",
         "description": "foo",
       },
+      "multibyteKey": {
+        "message": MULTIBYTE_STRING,
+      },
     }),
 
     "content.js": function() {
@@ -86,6 +93,7 @@ let extensionData = {
     "bar.CsS": "body { __MSG_foo__; }",
     "foo.txt": "body { __MSG_foo__; }",
     "locale.css": '* { content: "__MSG_@@ui_locale__ __MSG_@@bidi_dir__ __MSG_@@bidi_reversed_dir__ __MSG_@@bidi_start_edge__ __MSG_@@bidi_end_edge__" }',
+    "multibyte.css": getCSS("__MSG_multibyteKey__", MULTIBYTE_STRING),
   },
 };
 
@@ -94,11 +102,11 @@ async function test_i18n_css(options = {}) {
   let extension = ExtensionTestUtils.loadExtension(extensionData);
 
   await extension.startup();
-  let cssURL = await extension.awaitMessage("ready");
+  let baseURL = await extension.awaitMessage("ready");
 
   let contentPage = await ExtensionTestUtils.loadContentPage(`${BASE_URL}/file_sample.html`);
 
-  let css = await contentPage.fetch(cssURL);
+  let css = await contentPage.fetch(baseURL + "foo.css");
 
   equal(css, "body { max-width: 42px; }", "CSS file localized in mochitest scope");
 
@@ -106,11 +114,12 @@ async function test_i18n_css(options = {}) {
 
   equal(maxWidth, "42px", "stylesheet correctly applied");
 
-  cssURL = cssURL.replace(/foo.css$/, "locale.css");
-
-  css = await contentPage.fetch(cssURL);
+  css = await contentPage.fetch(baseURL + "locale.css");
   equal(css, '* { content: "en-US ltr rtl left right" }', "CSS file localized in mochitest scope");
 
+  css = await contentPage.fetch(baseURL + "multibyte.css");
+  equal(css, getCSS(MULTIBYTE_STRING, MULTIBYTE_STRING), "CSS file contains multibyte string");
+
   await contentPage.close();
 
   // We don't currently have a good way to mock this.
@@ -124,7 +133,7 @@ async function test_i18n_css(options = {}) {
     Services.locale.setRequestedLocales(["he"]);
     Preferences.set(DIR, 1);
 
-    css = await fetch(cssURL);
+    css = await fetch(baseURL + "locale.css");
     equal(css, '* { content: "he rtl ltr right left" }', "CSS file localized in mochitest scope");
 
     Services.locale.setRequestedLocales(origReqLocales);

toolkit/components/extensions/test/xpcshell/test_locale_converter.js

@@ -129,5 +129,5 @@ add_task(async function testInvalidUUID() {
 add_task(async function testEmptyStream() {
   let stream = StringStream("");
   let resultStream = convService.convert(stream, FROM_TYPE, TO_TYPE, URI);
-  equal(resultStream.data, "");
+  equal(resultStream.available(), 0, "Size of output stream should match size of input stream");
 });

toolkit/components/utils/simpleServices.js

@@ -25,6 +25,9 @@ XPCOMUtils.defineLazyServiceGetter(this, "catMan", "@mozilla.org/categorymanager
                                    "nsICategoryManager");
 XPCOMUtils.defineLazyServiceGetter(this, "streamConv", "@mozilla.org/streamConverters;1",
                                    "nsIStreamConverterService");
+const ArrayBufferInputStream = Components.Constructor(
+  "@mozilla.org/io/arraybuffer-input-stream;1",
+  "nsIArrayBufferInputStream", "setData");
 
 /*
  * This class provides a stream filter for locale messages in CSS files served
@@ -68,21 +71,18 @@ AddonLocalizationConverter.prototype = {
   },
 
   convertToStream(aAddon, aString) {
-    let stream = Cc["@mozilla.org/io/string-input-stream;1"]
-      .createInstance(Ci.nsIStringInputStream);
-
-    stream.data = aAddon.localize(aString);
-    return stream;
+    aString = aAddon.localize(aString);
+    let bytes = new TextEncoder().encode(aString).buffer;
+    return new ArrayBufferInputStream(bytes, 0, bytes.byteLength);
   },
 
   convert(aStream, aFromType, aToType, aContext) {
     this.checkTypes(aFromType, aToType);
     let addon = this.getAddon(aContext);
 
-    let string = (
-      aStream.available() ?
-      NetUtil.readInputStreamToString(aStream, aStream.available()) : ""
-    );
+    let count = aStream.available();
+    let string = count ?
+      new TextDecoder().decode(NetUtil.readInputStream(aStream, count)) : "";
     return this.convertToStream(addon, string);
   },
 
@@ -94,20 +94,23 @@ AddonLocalizationConverter.prototype = {
 
   onStartRequest(aRequest, aContext) {
     this.parts = [];
+    this.decoder = new TextDecoder();
   },
 
   onDataAvailable(aRequest, aContext, aInputStream, aOffset, aCount) {
-    this.parts.push(NetUtil.readInputStreamToString(aInputStream, aCount));
+    let bytes = NetUtil.readInputStream(aInputStream, aCount);
+    this.parts.push(this.decoder.decode(bytes, {stream: true}));
   },
 
   onStopRequest(aRequest, aContext, aStatusCode) {
     try {
       this.listener.onStartRequest(aRequest, null);
       if (Components.isSuccessCode(aStatusCode)) {
+        this.parts.push(this.decoder.decode());
         let string = this.parts.join("");
         let stream = this.convertToStream(this.addon, string);
 
-        this.listener.onDataAvailable(aRequest, null, stream, 0, stream.data.length);
+        this.listener.onDataAvailable(aRequest, null, stream, 0, stream.available());
       }
     } catch (e) {
       aStatusCode = e.result || Cr.NS_ERROR_FAILURE;