/* Any copyright is dedicated to the Public Domain.
http://creativecommons.org/publicdomain/zero/1.0/ */
"use strict";
/**
* @type {import("../../../ml/content/EngineProcess.sys.mjs")}
*/
const { EngineProcess } = ChromeUtils.importESModule(
"chrome://global/content/ml/EngineProcess.sys.mjs"
);
const { TranslationsPanelShared } = ChromeUtils.importESModule(
"chrome://browser/content/translations/TranslationsPanelShared.sys.mjs"
);
const { TranslationsUtils } = ChromeUtils.importESModule(
"chrome://global/content/translations/TranslationsUtils.mjs"
);
// Avoid about:blank's non-standard behavior.
const BLANK_PAGE =
"data:text/html;charset=utf-8,BlankBlank page";
const URL_COM_PREFIX = "https://example.com/browser/";
const URL_ORG_PREFIX = "https://example.org/browser/";
const CHROME_URL_PREFIX = "chrome://mochitests/content/browser/";
const DIR_PATH = "toolkit/components/translations/tests/browser/";
const ENGLISH_PAGE_URL =
URL_COM_PREFIX + DIR_PATH + "translations-tester-en.html";
const SPANISH_PAGE_URL =
URL_COM_PREFIX + DIR_PATH + "translations-tester-es.html";
const FRENCH_PAGE_URL =
URL_COM_PREFIX + DIR_PATH + "translations-tester-fr.html";
const SPANISH_PAGE_URL_2 =
URL_COM_PREFIX + DIR_PATH + "translations-tester-es-2.html";
const SPANISH_PAGE_URL_DOT_ORG =
URL_ORG_PREFIX + DIR_PATH + "translations-tester-es.html";
const NO_LANGUAGE_URL =
URL_COM_PREFIX + DIR_PATH + "translations-tester-no-tag.html";
const PDF_TEST_PAGE_URL =
URL_COM_PREFIX + DIR_PATH + "translations-tester-pdf-file.pdf";
const SELECT_TEST_PAGE_URL =
URL_COM_PREFIX + DIR_PATH + "translations-tester-select.html";
const TEXT_CLEANING_URL =
URL_COM_PREFIX + DIR_PATH + "translations-text-cleaning.html";
const SPANISH_BENCHMARK_PAGE_URL =
URL_COM_PREFIX + DIR_PATH + "translations-bencher-es.html";
const PIVOT_LANGUAGE = "en";
const LANGUAGE_PAIRS = [
{ fromLang: PIVOT_LANGUAGE, toLang: "es" },
{ fromLang: "es", toLang: PIVOT_LANGUAGE },
{ fromLang: PIVOT_LANGUAGE, toLang: "fr" },
{ fromLang: "fr", toLang: PIVOT_LANGUAGE },
{ fromLang: PIVOT_LANGUAGE, toLang: "uk" },
{ fromLang: "uk", toLang: PIVOT_LANGUAGE },
];
const TRANSLATIONS_PERMISSION = "translations";
const ALWAYS_TRANSLATE_LANGS_PREF =
"browser.translations.alwaysTranslateLanguages";
const NEVER_TRANSLATE_LANGS_PREF =
"browser.translations.neverTranslateLanguages";
const USE_LEXICAL_SHORTLIST_PREF = "browser.translations.useLexicalShortlist";
/**
* Generates a sorted list of Translation model file names for the given language pairs.
*
* @param {Array<{ fromLang: string, toLang: string }>} languagePairs - An array of language pair objects.
*
* @returns {string[]} A sorted array of translation model file names.
*/
function languageModelNames(languagePairs) {
return languagePairs
.flatMap(({ fromLang, toLang }) => [
`model.${fromLang}${toLang}.intgemm.alphas.bin`,
`vocab.${fromLang}${toLang}.spm`,
...(Services.prefs.getBoolPref(USE_LEXICAL_SHORTLIST_PREF)
? [`lex.50.50.${fromLang}${toLang}.s2t.bin`]
: []),
])
.sort();
}
/**
* The mochitest runs in the parent process. This function opens up a new tab,
* opens up about:translations, and passes the test requirements into the content process.
*
* @template T
*
* @param {object} options
*
* @param {T} options.dataForContent
* The data must support structural cloning and will be passed into the
* content process.
*
* @param {boolean} [options.disabled]
* Disable the panel through a pref.
*
* @param {Array<{ fromLang: string, toLang: string }>} options.languagePairs
* The translation languages pairs to mock for the test.
*
* @param {Array<[string, string]>} options.prefs
* Prefs to push on for the test.
*
* @param {boolean} [options.autoDownloadFromRemoteSettings=true]
* Initiate the mock model downloads when this function is invoked instead of
* waiting for the resolveDownloads or rejectDownloads to be externally invoked
*
* @returns {object} object
*
* @returns {(args: { dataForContent: T, selectors: Record }) => Promise} object.runInPage
* This function must not capture any values, as it will be cloned in the content process.
* Any required data should be passed in using the "dataForContent" parameter. The
* "selectors" property contains any useful selectors for the content.
*
* @returns {() => Promise} object.cleanup
*
* @returns {(count: number) => Promise} object.resolveDownloads
*
* @returns {(count: number) => Promise} object.rejectDownloads
*/
async function openAboutTranslations({
dataForContent,
disabled,
languagePairs = LANGUAGE_PAIRS,
prefs,
autoDownloadFromRemoteSettings = false,
} = {}) {
await SpecialPowers.pushPrefEnv({
set: [
// Enabled by default.
["browser.translations.enable", !disabled],
["browser.translations.logLevel", "All"],
["browser.translations.mostRecentTargetLanguages", ""],
[USE_LEXICAL_SHORTLIST_PREF, false],
...(prefs ?? []),
],
});
/**
* Collect any relevant selectors for the page here.
*/
const selectors = {
pageHeader: '[data-l10n-id="about-translations-header"]',
fromLanguageSelect: "select#language-from",
toLanguageSelect: "select#language-to",
languageSwapButton: "button#language-swap",
translationTextarea: "textarea#translation-from",
translationResult: "#translation-to",
translationResultBlank: "#translation-to-blank",
translationInfo: "#translation-info",
translationResultsPlaceholder: "#translation-results-placeholder",
noSupportMessage: "[data-l10n-id='about-translations-no-support']",
};
// Start the tab at a blank page.
let tab = await BrowserTestUtils.openNewForegroundTab(
gBrowser,
BLANK_PAGE,
true // waitForLoad
);
const { removeMocks, remoteClients } = await createAndMockRemoteSettings({
languagePairs,
autoDownloadFromRemoteSettings,
});
// Now load the about:translations page, since the actor could be mocked.
BrowserTestUtils.startLoadingURIString(
tab.linkedBrowser,
"about:translations"
);
await BrowserTestUtils.browserLoaded(tab.linkedBrowser);
/**
* @param {number} count - Count of the language pairs expected.
*/
const resolveDownloads = async count => {
await remoteClients.translationsWasm.resolvePendingDownloads(1);
await remoteClients.translationModels.resolvePendingDownloads(
downloadedFilesPerLanguagePair() * count
);
};
/**
* @param {number} count - Count of the language pairs expected.
*/
const rejectDownloads = async count => {
await remoteClients.translationsWasm.rejectPendingDownloads(1);
await remoteClients.translationModels.rejectPendingDownloads(
downloadedFilesPerLanguagePair() * count
);
};
return {
runInPage(callback) {
return ContentTask.spawn(
tab.linkedBrowser,
{ dataForContent, selectors }, // Data to inject.
callback
);
},
async cleanup() {
await loadBlankPage();
BrowserTestUtils.removeTab(tab);
await removeMocks();
await EngineProcess.destroyTranslationsEngine();
await SpecialPowers.popPrefEnv();
TestTranslationsTelemetry.reset();
Services.fog.testResetFOG();
},
resolveDownloads,
rejectDownloads,
};
}
/**
* Naively prettify's html based on the opening and closing tags. This is not robust
* for general usage, but should be adequate for these tests.
*
* @param {string} html
* @returns {string}
*/
function naivelyPrettify(html) {
let result = "";
let indent = 0;
function addText(actualEndIndex) {
const text = html.slice(startIndex, actualEndIndex).trim();
if (text) {
for (let i = 0; i < indent; i++) {
result += " ";
}
result += text + "\n";
}
startIndex = actualEndIndex;
}
let startIndex = 0;
let endIndex = 0;
for (; endIndex < html.length; endIndex++) {
if (
html[endIndex] === " " ||
html[endIndex] === "\t" ||
html[endIndex] === "n"
) {
// Skip whitespace.
// " foobar
"
// ^^^
startIndex = endIndex;
continue;
}
// Find all of the text.
// "foobar
"
// ^^^^^^
while (endIndex < html.length && html[endIndex] !== "<") {
endIndex++;
}
addText(endIndex);
if (html[endIndex] === "<") {
if (html[endIndex + 1] === "/") {
// "foobar
"
// ^
while (endIndex < html.length && html[endIndex] !== ">") {
endIndex++;
}
indent--;
addText(endIndex + 1);
} else {
// "foobar
"
// ^
while (endIndex < html.length && html[endIndex] !== ">") {
endIndex++;
}
// "foobar
"
// ^
addText(endIndex + 1);
indent++;
}
}
}
return result.trim();
}
/**
* Recursively transforms all child nodes to have uppercased text.
*
* @param {Node} node
*/
function upperCaseNode(node) {
if (typeof node.nodeValue === "string") {
node.nodeValue = node.nodeValue.toUpperCase();
}
for (const childNode of node.childNodes) {
upperCaseNode(childNode);
}
}
/**
* Recursively transforms all child nodes to have diacriticized text. This is useful
* to spot multiple translations.
*
* @param {Node} node
*/
function diacriticizeNode(node) {
if (typeof node.nodeValue === "string") {
let result = "";
for (let i = 0; i < node.nodeValue.length; i++) {
const ch = node.nodeValue[i];
result += ch;
if ("abcdefghijklmnopqrstuvwxyz".includes(ch.toLowerCase())) {
result += "\u0305";
}
}
node.nodeValue = result;
}
for (const childNode of node.childNodes) {
diacriticizeNode(childNode);
}
}
/**
* Creates a mocked message port for translations.
*
* @returns {MessagePort} This is mocked
*/
function createMockedTranslatorPort(transformNode = upperCaseNode, delay = 0) {
const parser = new DOMParser();
const mockedPort = {
async postMessage(message) {
// Make this response async.
await TestUtils.waitForTick();
switch (message.type) {
case "TranslationsPort:GetEngineStatusRequest":
mockedPort.onmessage({
data: {
type: "TranslationsPort:GetEngineStatusResponse",
status: "ready",
},
});
break;
case "TranslationsPort:TranslationRequest": {
const { translationId, sourceText } = message;
const translatedDoc = parser.parseFromString(sourceText, "text/html");
transformNode(translatedDoc.body);
if (delay) {
await new Promise(resolve => setTimeout(resolve, delay));
}
mockedPort.onmessage({
data: {
type: "TranslationsPort:TranslationResponse",
targetText: translatedDoc.body.innerHTML,
translationId,
},
});
}
}
},
};
return mockedPort;
}
class TranslationResolver {
resolvers = Promise.withResolvers();
resolveCount = 0;
getPromise() {
return this.resolvers.promise;
}
}
/**
* Creates a mocked message port for translations.
*
* @returns {MessagePort} This is mocked
*/
function createControlledTranslatorPort() {
const parser = new DOMParser();
const canceledTranslations = new Set();
let resolvers = Promise.withResolvers();
let translationCount = 0;
async function resolveRequests() {
info("Resolving all pending translation requests");
await TestUtils.waitForTick();
resolvers.resolve();
resolvers = Promise.withResolvers();
await TestUtils.waitForTick();
const count = translationCount;
translationCount = 0;
return count;
}
const mockedTranslatorPort = {
async postMessage(message) {
switch (message.type) {
case "TranslationsPort:CancelSingleTranslation":
info("Canceling translation id:" + message.translationId);
canceledTranslations.add(message.translationId);
break;
case "TranslationsPort:GetEngineStatusRequest":
mockedTranslatorPort.onmessage({
data: {
type: "TranslationsPort:GetEngineStatusResponse",
status: "ready",
},
});
break;
case "TranslationsPort:TranslationRequest": {
const { translationId, sourceText } = message;
// Create a short debug version of the text.
let debugText = sourceText.trim().replaceAll("\n", "");
if (debugText.length > 50) {
debugText = debugText.slice(0, 50) + "...";
}
info(
`Translation requested (id:${message.translationId}) "${debugText}"`
);
await resolvers.promise;
if (canceledTranslations.has(translationId)) {
info("Cancelled translation id:" + translationId);
} else {
info(
"Translation completed, responding id:" + message.translationId
);
translationCount++;
const translatedDoc = parser.parseFromString(
sourceText,
"text/html"
);
diacriticizeNode(translatedDoc.body);
const targetText =
translatedDoc.body.innerHTML.trim() + ` (id:${translationId})`;
info("Translation response: " + targetText.replaceAll("\n", ""));
mockedTranslatorPort.onmessage({
data: {
type: "TranslationsPort:TranslationResponse",
targetText,
translationId,
},
});
}
}
}
},
};
return { mockedTranslatorPort, resolveRequests };
}
/**
* @type {typeof import("../../content/translations-document.sys.mjs")}
*/
const { TranslationsDocument, LRUCache } = ChromeUtils.importESModule(
"chrome://global/content/translations/translations-document.sys.mjs"
);
/**
* @param {string} html
* @param {{
* mockedTranslatorPort?: (message: string) => Promise,
* mockedReportVisibleChange?: () => void
* }} [options]
*/
async function createTranslationsDoc(html, options) {
await SpecialPowers.pushPrefEnv({
set: [
["browser.translations.enable", true],
["browser.translations.logLevel", "All"],
[USE_LEXICAL_SHORTLIST_PREF, false],
],
});
const parser = new DOMParser();
const document = parser.parseFromString(html, "text/html");
// For some reason, the document here from the DOMParser is "display: flex" by
// default. Ensure that it is "display: block" instead, otherwise the children of the
// will not be "display: inline".
document.body.style.display = "block";
const translate = () => {
info("Creating the TranslationsDocument.");
return new TranslationsDocument(
document,
"en",
"EN",
0, // This is a fake innerWindowID
options?.mockedTranslatorPort ?? createMockedTranslatorPort(),
() => {
throw new Error("Cannot request a new port");
},
options?.mockedReportVisibleChange ?? (() => {}),
performance.now(),
() => performance.now(),
new LRUCache()
);
};
/**
* Test utility to check that the document matches the expected markup
*
* @param {string} message
* @param {string} html
*/
async function htmlMatches(message, html, element = document.body) {
const expected = naivelyPrettify(html);
try {
await waitForCondition(
() => naivelyPrettify(element.innerHTML) === expected,
"Waiting for HTML to match."
);
ok(true, message);
} catch (error) {
console.error(error);
// Provide a nice error message.
const actual = naivelyPrettify(element.innerHTML);
ok(
false,
`${message}\n\nExpected HTML:\n\n${expected}\n\nActual HTML:\n\n${actual}\n\n`
);
}
}
function cleanup() {
SpecialPowers.popPrefEnv();
}
return { htmlMatches, cleanup, translate, document };
}
/**
* Perform a double requestAnimationFrame, which is used by the TranslationsDocument
* to handle mutations.
*
* @param {Document} doc
*/
function doubleRaf(doc) {
return new Promise(resolve => {
doc.ownerGlobal.requestAnimationFrame(() => {
doc.ownerGlobal.requestAnimationFrame(() => {
resolve(
// Wait for a tick to be after anything that resolves with a double rAF.
TestUtils.waitForTick()
);
});
});
});
}
/**
* This mocked translator reports on the batching of calls by replacing the text
* with a letter. Each call of the function moves the letter forward alphabetically.
*
* So consecutive calls would transform things like:
* "First translation" -> "aaaa aaaaaaaaa"
* "Second translation" -> "bbbbb bbbbbbbbb"
* "Third translation" -> "cccc ccccccccc"
*
* This can visually show what the translation batching behavior looks like.
*
* @returns {MessagePort} A mocked port.
*/
function createBatchedMockedTranslatorPort() {
let letter = "a";
/**
* @param {Node} node
*/
function transformNode(node) {
if (typeof node.nodeValue === "string") {
node.nodeValue = node.nodeValue.replace(/\w/g, letter);
}
for (const childNode of node.childNodes) {
transformNode(childNode);
}
}
return createMockedTranslatorPort(node => {
transformNode(node);
letter = String.fromCodePoint(letter.codePointAt(0) + 1);
});
}
/**
* This mocked translator reorders Nodes to be in alphabetical order, and then
* uppercases the text. This allows for testing the reordering behavior of the
* translation engine.
*
* @returns {MessagePort} A mocked port.
*/
function createdReorderingMockedTranslatorPort() {
/**
* @param {Node} node
*/
function transformNode(node) {
if (typeof node.nodeValue === "string") {
node.nodeValue = node.nodeValue.toUpperCase();
}
const nodes = [...node.childNodes];
nodes.sort((a, b) =>
(a.textContent?.trim() ?? "").localeCompare(b.textContent?.trim() ?? "")
);
for (const childNode of nodes) {
childNode.remove();
}
for (const childNode of nodes) {
// Re-append in sorted order.
node.appendChild(childNode);
transformNode(childNode);
}
}
return createMockedTranslatorPort(transformNode);
}
/**
* @returns {import("../../actors/TranslationsParent.sys.mjs").TranslationsParent}
*/
function getTranslationsParent() {
return TranslationsParent.getTranslationsActor(gBrowser.selectedBrowser);
}
/**
* Closes all open panels and menu popups related to Translations.
*
* @param {ChromeWindow} [win]
*/
async function closeAllOpenPanelsAndMenus(win) {
await closeFullPagePanelSettingsMenuIfOpen(win);
await closeFullPageTranslationsPanelIfOpen(win);
await closeSelectPanelSettingsMenuIfOpen(win);
await closeSelectTranslationsPanelIfOpen(win);
await closeContextMenuIfOpen(win);
}
/**
* Closes the popup element with the given Id if it is open.
*
* @param {string} popupElementId
* @param {ChromeWindow} [win]
*/
async function closePopupIfOpen(popupElementId, win = window) {
await waitForCondition(async () => {
const popupElement = win.document.getElementById(popupElementId);
if (!popupElement) {
return true;
}
if (popupElement.state === "closed") {
return true;
}
let popuphiddenPromise = BrowserTestUtils.waitForEvent(
popupElement,
"popuphidden"
);
popupElement.hidePopup();
PanelMultiView.hidePopup(popupElement);
await popuphiddenPromise;
return false;
});
}
/**
* Closes the context menu if it is open.
*
* @param {ChromeWindow} [win]
*/
async function closeContextMenuIfOpen(win) {
await closePopupIfOpen("contentAreaContextMenu", win);
}
/**
* Closes the full-page translations panel settings menu if it is open.
*
* @param {ChromeWindow} [win]
*/
async function closeFullPagePanelSettingsMenuIfOpen(win) {
await closePopupIfOpen(
"full-page-translations-panel-settings-menupopup",
win
);
}
/**
* Closes the select translations panel settings menu if it is open.
*
* @param {ChromeWindow} [win]
*/
async function closeSelectPanelSettingsMenuIfOpen(win) {
await closePopupIfOpen("select-translations-panel-settings-menupopup", win);
}
/**
* Closes the translations panel if it is open.
*
* @param {ChromeWindow} [win]
*/
async function closeFullPageTranslationsPanelIfOpen(win) {
await closePopupIfOpen("full-page-translations-panel", win);
}
/**
* Closes the translations panel if it is open.
*
* @param {ChromeWindow} [win]
*/
async function closeSelectTranslationsPanelIfOpen(win) {
await closePopupIfOpen("select-translations-panel", win);
}
/**
* This is for tests that don't need a browser page to run.
*/
async function setupActorTest({
languagePairs,
prefs,
autoDownloadFromRemoteSettings = false,
}) {
await SpecialPowers.pushPrefEnv({
set: [
// Enabled by default.
["browser.translations.enable", true],
["browser.translations.logLevel", "All"],
[USE_LEXICAL_SHORTLIST_PREF, false],
...(prefs ?? []),
],
});
const { remoteClients, removeMocks } = await createAndMockRemoteSettings({
languagePairs,
autoDownloadFromRemoteSettings,
});
// Create a new tab so each test gets a new actor, and doesn't re-use the old one.
const tab = await BrowserTestUtils.openNewForegroundTab(
gBrowser,
ENGLISH_PAGE_URL,
true // waitForLoad
);
const actor = getTranslationsParent();
return {
actor,
remoteClients,
async cleanup() {
await closeAllOpenPanelsAndMenus();
await loadBlankPage();
await EngineProcess.destroyTranslationsEngine();
BrowserTestUtils.removeTab(tab);
await removeMocks();
TestTranslationsTelemetry.reset();
return SpecialPowers.popPrefEnv();
},
};
}
/**
* Creates and mocks remote settings for translations.
*
* @param {object} options - The options for creating and mocking remote settings.
* @param {Array<{fromLang: string, toLang: string}>} [options.languagePairs=LANGUAGE_PAIRS]
* - The language pairs to be used.
* @param {boolean} [options.useMockedTranslator=true]
* - Whether to use a mocked translator.
* @param {boolean} [options.autoDownloadFromRemoteSettings=false]
* - Whether to automatically download from remote settings.
*
* @returns {Promise