corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: remotesettings disable, use offline dumps

Browse Source
This commit is contained in:
Alex Kontos
2025-07-24 15:11:09 +01:00
parent 79f5b2572b
commit ea8e999826
8 changed files with 164 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
browser/components/BrowserGlue.sys.mjs
View File

@@ -1401,6 +1401,11 @@ BrowserGlue.prototype = {
lazy.RemoteSecuritySettings.init();
},
function RemoteSettingsPollChanges() {
// Support clients that use the "sync" event or "remote-settings:changes-poll-end".
lazy.RemoteSettings.pollChanges({ trigger: "timer" });
},
function searchBackgroundChecks() {
Services.search.runBackgroundChecks();
},

3
mobile/shared/chrome/geckoview/geckoview.js
View File

@@ -21,6 +21,7 @@ ChromeUtils.defineESModuleGetters(this, {
InitializationTracker: "resource://gre/modules/GeckoViewTelemetry.sys.mjs",
RemoteSecuritySettings:
"resource://gre/modules/psm/RemoteSecuritySettings.sys.mjs",
RemoteSettings: "resource://services-settings/remote-settings.sys.mjs",
SafeBrowsing: "resource://gre/modules/SafeBrowsing.sys.mjs",
CaptchaDetectionPingUtils:
"resource://gre/modules/CaptchaDetectionPingUtils.sys.mjs",
@@ -928,6 +929,8 @@ function startup() {
// submits the ping if it has data and has been about 24 hours since the
// last submission.
CaptchaDetectionPingUtils.init();
RemoteSettings.pollChanges({ trigger: "timer" });
});
// This should always go last, since the idle tasks (except for the ones with

11
security/manager/ssl/RemoteSecuritySettings.sys.mjs
View File

@@ -346,7 +346,7 @@ class IntermediatePreloads {
lazy.log.debug(
`There are ${waiting.length} intermediates awaiting download.`
);
if (!waiting.length) {
if (!waiting.length || !Services.prefs.getBoolPref(INTERMEDIATES_ENABLED_PREF, true) || !this.client) {
// Nothing to do.
Services.obs.notifyObservers(
null,
@@ -441,6 +441,11 @@ class IntermediatePreloads {
async maybeDownloadAttachment(record) {
let result = { record, cert: null, subject: null };
// Early return if intermediates are disabled or client doesn't exist
if (!Services.prefs.getBoolPref(INTERMEDIATES_ENABLED_PREF, true) || !this.client) {
return result;
}
let dataAsString = null;
try {
let { buffer } = await this.client.attachments.download(record, {
@@ -642,6 +647,10 @@ class CRLiteFilters {
lazy.log.debug("filtersToDownload:", filtersToDownload);
let filtersDownloaded = [];
for (let filter of filtersToDownload) {
// Skip download if CRLite filters are disabled or client doesn't exist
if (!Services.prefs.getBoolPref(CRLITE_FILTERS_ENABLED_PREF, true) || !this.client) {
continue;
}
try {
let attachment = await this.client.attachments.downloadAsBytes(filter);
let bytes = new Uint8Array(attachment);

84
services/settings/Attachments.sys.mjs
View File

@@ -338,6 +338,30 @@ export class Downloader {
fallbackToDump = false,
avoidDownload = false,
} = options || {};
// Detect translation files based on record properties (collection is often undefined)
const isTranslationFile = record && (
record.fileType === "model" ||
record.fileType === "lex" ||
record.fileType === "wasm" ||
record.fileType === "vocab" ||
(record.fromLang && record.toLang) ||
(record.name && (
record.name.includes("model.") ||
record.name.includes("lex.") ||
record.name.includes("vocab.") ||
record.name.includes("bergamot")
)) ||
(record.attachment?.filename && (
record.attachment.filename.endsWith(".wasm") ||
record.attachment.filename.includes("bergamot")
))
);
// Force avoidDownload for non-translation files
if (!isTranslationFile) {
avoidDownload = true;
}
if (!attachmentId) {
// Check for pre-condition. This should not happen, but it is explicitly
// checked to avoid mixing up attachments, which could be dangerous.
@@ -510,7 +534,7 @@ export class Downloader {
* @throws {Downloader.BadContentError} if the downloaded content integrity is not valid.
* @returns {ArrayBuffer} the file content.
*/
async downloadAsBytes(record, options = {}) {
async downloadAsBytes(record, options) {
const {
attachment: { location, hash, size },
} = record;
@@ -519,8 +543,66 @@ export class Downloader {
try {
baseURL = await lazy.Utils.baseAttachmentsURL();
} catch (error) {
// For translation files only, fallback to Mozilla's server if Waterfox's is empty
const isTranslationFile = record && (
record.fileType === "model" ||
record.fileType === "lex" ||
record.fileType === "wasm" ||
record.fileType === "vocab" ||
(record.fromLang && record.toLang) ||
(record.name && (
record.name.includes("model.") ||
record.name.includes("lex.") ||
record.name.includes("vocab.") ||
record.name.includes("bergamot")
)) ||
(record.attachment?.filename && (
record.attachment.filename.endsWith(".wasm") ||
record.attachment.filename.includes("bergamot")
))
);
if (isTranslationFile) {
// Use Mozilla's CDN for translation files when server URL is empty
baseURL = "https://firefox.settings.services.mozilla.com/v1/";
const resp = await lazy.Utils.fetch(baseURL);
const serverInfo = await resp.json();
baseURL = serverInfo.capabilities.attachments.base_url;
if (!baseURL.endsWith("/")) {
baseURL += "/";
}
} else {
throw new Downloader.ServerInfoError(error);
}
}
// Additional check: if baseURL is still empty and it's a translation file
if (!baseURL || baseURL === "/") {
const isTranslationFile = record && (
record.fileType === "model" ||
record.fileType === "lex" ||
record.fileType === "wasm" ||
record.fileType === "vocab" ||
(record.fromLang && record.toLang) ||
(record.name && (
record.name.includes("model.") ||
record.name.includes("lex.") ||
record.name.includes("vocab.") ||
record.name.includes("bergamot")
)) ||
(record.attachment?.filename && (
record.attachment.filename.endsWith(".wasm") ||
record.attachment.filename.includes("bergamot")
))
);
if (isTranslationFile) {
// Hardcode Mozilla's CDN URL for translations
baseURL = "https://cdn.settings.services.mozilla.com/";
} else {
throw new Downloader.ServerInfoError("No server URL configured");
}
}
const remoteFileUrl = baseURL + location;

46
services/settings/RemoteSettingsClient.sys.mjs
View File

@@ -437,11 +437,19 @@ export class RemoteSettingsClient extends EventEmitter {
order = "", // not sorted by default.
dumpFallback = true,
emptyListFallback = true,
forceSync = false,
loadDumpIfNewer = true,
syncIfEmpty = true,
} = options;
let { verifySignature = false } = options;
const hasLocalDump = await lazy.Utils.hasLocalDump(
this.bucketName,
this.collectionName
);
if (!hasLocalDump) {
return [];
}
const forceSync = false;
const syncIfEmpty = true;
let verifySignature = false;
const hasParallelCall = !!this._importingPromise;
let data;
@@ -621,6 +629,10 @@ export class RemoteSettingsClient extends EventEmitter {
* @param {Object} options See #maybeSync() options.
*/
async sync(options) {
if (AppConstants.MOZ_APP_VERSION) {
return;
}
if (lazy.Utils.shouldSkipRemoteActivityDueToTests) {
return;
}
@@ -693,7 +705,7 @@ export class RemoteSettingsClient extends EventEmitter {
let thrownError = null;
try {
// If network is offline, we can't synchronize.
if (lazy.Utils.isOffline) {
if (!AppConstants.MOZ_APP_VERSION && lazy.Utils.isOffline) {
throw new RemoteSettingsClient.NetworkOfflineError();
}
@@ -1079,14 +1091,8 @@ export class RemoteSettingsClient extends EventEmitter {
options = {}
) {
const { retry = false } = options;
const since = retry || !localTimestamp ? undefined : `"${localTimestamp}"`;
// Fetch collection metadata and list of changes from server.
lazy.console.debug(
`${this.identifier} Fetch changes from server (expected=${expectedTimestamp}, since=${since})`
);
const { metadata, remoteTimestamp, remoteRecords } =
await this._fetchChangeset(expectedTimestamp, since);
let metadata, remoteTimestamp;
// We build a sync result, based on remote changes.
const syncResult = {
@@ -1095,24 +1101,20 @@ export class RemoteSettingsClient extends EventEmitter {
updated: [],
deleted: [],
};
// If data wasn't changed, return empty sync result.
// This can happen when we update the signature but not the data.
lazy.console.debug(
`${this.identifier} local timestamp: ${localTimestamp}, remote: ${remoteTimestamp}`
);
if (localTimestamp && remoteTimestamp < localTimestamp) {
try {
await this._importJSONDump();
} catch (e) {
return syncResult;
}
await this.db.importChanges(metadata, remoteTimestamp, remoteRecords, {
clear: retry,
});
// Read the new local data, after updating.
const newLocal = await this.db.list();
const newRecords = newLocal.map(r => this._cleanLocalFields(r));
// And verify the signature on what is now stored.
if (this.verifySignature) {
if (metadata === undefined) {
// When working only with dumps, we do not have signatures.
} else if (this.verifySignature) {
try {
await this.validateCollectionSignature(
newRecords,

2
services/settings/dumps/gen_last_modified.py
View File

@@ -63,8 +63,10 @@ def main(output):
dumps_locations = []
if buildconfig.substs["MOZ_BUILD_APP"] == "browser":
dumps_locations += ["services/settings/dumps/"]
dumps_locations += ["services/settings/static-dumps/"]
elif buildconfig.substs["MOZ_BUILD_APP"] == "mobile/android":
dumps_locations += ["services/settings/dumps/"]
dumps_locations += ["services/settings/static-dumps/"]
elif buildconfig.substs["MOZ_BUILD_APP"] == "mobile/ios":
dumps_locations += ["services/settings/dumps/"]
elif buildconfig.substs["MOZ_BUILD_APP"] == "comm/mail":

34
services/settings/remote-settings.sys.mjs
View File

@@ -103,6 +103,7 @@ export async function jexlFilterFunc(entry, environment, collectionName) {
function remoteSettingsFunction() {
const _clients = new Map();
let _invalidatePolling = false;
let _initialized = false;
// If not explicitly specified, use the default signer.
const defaultOptions = {
@@ -304,6 +305,39 @@ function remoteSettingsFunction() {
trigger = "manual",
full = false,
} = {}) => {
if (AppConstants.MOZ_APP_VERSION) {
// Called multiple times on GeckoView due to bug 1730026
if (_initialized) {
return;
}
_initialized = true;
let importedFromDump = false;
for (const client of _clients.values()) {
const hasLocalDump = await lazy.Utils.hasLocalDump(
client.bucketName,
client.collectionName
);
if (hasLocalDump) {
const lastModified = await client.getLastModified();
const lastModifiedDump = await lazy.Utils.getLocalDumpLastModified(
client.bucketName,
client.collectionName
);
if (lastModified < lastModifiedDump) {
await client.maybeSync(lastModifiedDump, {
loadDump: true,
trigger,
});
importedFromDump = true;
}
}
}
if (importedFromDump) {
Services.obs.notifyObservers(null, "remote-settings:changes-poll-end");
}
return;
}
if (lazy.Utils.shouldSkipRemoteActivityDueToTests) {
return;
}

4
toolkit/modules/AppConstants.sys.mjs
View File

@@ -211,11 +211,11 @@ export var AppConstants = Object.freeze({
#ifdef MOZ_THUNDERBIRD
"https://thunderbird-settings.thunderbird.net/v1",
#else
"https://firefox.settings.services.mozilla.com/v1",
"",
#endif
REMOTE_SETTINGS_VERIFY_SIGNATURE:
#ifdef MOZ_THUNDERBIRD
#if defined(MOZ_THUNDERBIRD) || defined(MOZ_APP_VERSION)
false,
#else
true,