corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: remotesettings disable, use offline dumps

Browse Source
This commit is contained in:
Alex Kontos
2025-07-24 15:11:09 +01:00
parent 79f5b2572b
commit ea8e999826
8 changed files with 164 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
services/settings/Attachments.sys.mjs
View File

@@ -338,6 +338,30 @@ export class Downloader {
fallbackToDump = false,
avoidDownload = false,
} = options || {};
// Detect translation files based on record properties (collection is often undefined)
const isTranslationFile = record && (
record.fileType === "model" ||
record.fileType === "lex" ||
record.fileType === "wasm" ||
record.fileType === "vocab" ||
(record.fromLang && record.toLang) ||
(record.name && (
record.name.includes("model.") ||
record.name.includes("lex.") ||
record.name.includes("vocab.") ||
record.name.includes("bergamot")
)) ||
(record.attachment?.filename && (
record.attachment.filename.endsWith(".wasm") ||
record.attachment.filename.includes("bergamot")
))
);
// Force avoidDownload for non-translation files
if (!isTranslationFile) {
avoidDownload = true;
}
if (!attachmentId) {
// Check for pre-condition. This should not happen, but it is explicitly
// checked to avoid mixing up attachments, which could be dangerous.
@@ -510,7 +534,7 @@ export class Downloader {
* @throws {Downloader.BadContentError} if the downloaded content integrity is not valid.
* @returns {ArrayBuffer} the file content.
*/
async downloadAsBytes(record, options = {}) {
async downloadAsBytes(record, options) {
const {
attachment: { location, hash, size },
} = record;
@@ -519,7 +543,65 @@ export class Downloader {
try {
baseURL = await lazy.Utils.baseAttachmentsURL();
} catch (error) {
throw new Downloader.ServerInfoError(error);
// For translation files only, fallback to Mozilla's server if Waterfox's is empty
const isTranslationFile = record && (
record.fileType === "model" ||
record.fileType === "lex" ||
record.fileType === "wasm" ||
record.fileType === "vocab" ||
(record.fromLang && record.toLang) ||
(record.name && (
record.name.includes("model.") ||
record.name.includes("lex.") ||
record.name.includes("vocab.") ||
record.name.includes("bergamot")
)) ||
(record.attachment?.filename && (
record.attachment.filename.endsWith(".wasm") ||
record.attachment.filename.includes("bergamot")
))
);
if (isTranslationFile) {
// Use Mozilla's CDN for translation files when server URL is empty
baseURL = "https://firefox.settings.services.mozilla.com/v1/";
const resp = await lazy.Utils.fetch(baseURL);
const serverInfo = await resp.json();
baseURL = serverInfo.capabilities.attachments.base_url;
if (!baseURL.endsWith("/")) {
baseURL += "/";
}
} else {
throw new Downloader.ServerInfoError(error);
}
}
// Additional check: if baseURL is still empty and it's a translation file
if (!baseURL || baseURL === "/") {
const isTranslationFile = record && (
record.fileType === "model" ||
record.fileType === "lex" ||
record.fileType === "wasm" ||
record.fileType === "vocab" ||
(record.fromLang && record.toLang) ||
(record.name && (
record.name.includes("model.") ||
record.name.includes("lex.") ||
record.name.includes("vocab.") ||
record.name.includes("bergamot")
)) ||
(record.attachment?.filename && (
record.attachment.filename.endsWith(".wasm") ||
record.attachment.filename.includes("bergamot")
))
);
if (isTranslationFile) {
// Hardcode Mozilla's CDN URL for translations
baseURL = "https://cdn.settings.services.mozilla.com/";
} else {
throw new Downloader.ServerInfoError("No server URL configured");
}
}
const remoteFileUrl = baseURL + location;

46
services/settings/RemoteSettingsClient.sys.mjs
View File

@@ -437,11 +437,19 @@ export class RemoteSettingsClient extends EventEmitter {
order = "", // not sorted by default.
dumpFallback = true,
emptyListFallback = true,
forceSync = false,
loadDumpIfNewer = true,
syncIfEmpty = true,
} = options;
let { verifySignature = false } = options;
const hasLocalDump = await lazy.Utils.hasLocalDump(
this.bucketName,
this.collectionName
);
if (!hasLocalDump) {
return [];
}
const forceSync = false;
const syncIfEmpty = true;
let verifySignature = false;
const hasParallelCall = !!this._importingPromise;
let data;
@@ -621,6 +629,10 @@ export class RemoteSettingsClient extends EventEmitter {
* @param {Object} options See #maybeSync() options.
*/
async sync(options) {
if (AppConstants.MOZ_APP_VERSION) {
return;
}
if (lazy.Utils.shouldSkipRemoteActivityDueToTests) {
return;
}
@@ -693,7 +705,7 @@ export class RemoteSettingsClient extends EventEmitter {
let thrownError = null;
try {
// If network is offline, we can't synchronize.
if (lazy.Utils.isOffline) {
if (!AppConstants.MOZ_APP_VERSION && lazy.Utils.isOffline) {
throw new RemoteSettingsClient.NetworkOfflineError();
}
@@ -1079,14 +1091,8 @@ export class RemoteSettingsClient extends EventEmitter {
options = {}
) {
const { retry = false } = options;
const since = retry || !localTimestamp ? undefined : `"${localTimestamp}"`;
// Fetch collection metadata and list of changes from server.
lazy.console.debug(
`${this.identifier} Fetch changes from server (expected=${expectedTimestamp}, since=${since})`
);
const { metadata, remoteTimestamp, remoteRecords } =
await this._fetchChangeset(expectedTimestamp, since);
let metadata, remoteTimestamp;
// We build a sync result, based on remote changes.
const syncResult = {
@@ -1095,24 +1101,20 @@ export class RemoteSettingsClient extends EventEmitter {
updated: [],
deleted: [],
};
// If data wasn't changed, return empty sync result.
// This can happen when we update the signature but not the data.
lazy.console.debug(
`${this.identifier} local timestamp: ${localTimestamp}, remote: ${remoteTimestamp}`
);
if (localTimestamp && remoteTimestamp < localTimestamp) {
try {
await this._importJSONDump();
} catch (e) {
return syncResult;
}
await this.db.importChanges(metadata, remoteTimestamp, remoteRecords, {
clear: retry,
});
// Read the new local data, after updating.
const newLocal = await this.db.list();
const newRecords = newLocal.map(r => this._cleanLocalFields(r));
// And verify the signature on what is now stored.
if (this.verifySignature) {
if (metadata === undefined) {
// When working only with dumps, we do not have signatures.
} else if (this.verifySignature) {
try {
await this.validateCollectionSignature(
newRecords,

2
services/settings/dumps/gen_last_modified.py
View File

@@ -63,8 +63,10 @@ def main(output):
dumps_locations = []
if buildconfig.substs["MOZ_BUILD_APP"] == "browser":
dumps_locations += ["services/settings/dumps/"]
dumps_locations += ["services/settings/static-dumps/"]
elif buildconfig.substs["MOZ_BUILD_APP"] == "mobile/android":
dumps_locations += ["services/settings/dumps/"]
dumps_locations += ["services/settings/static-dumps/"]
elif buildconfig.substs["MOZ_BUILD_APP"] == "mobile/ios":
dumps_locations += ["services/settings/dumps/"]
elif buildconfig.substs["MOZ_BUILD_APP"] == "comm/mail":

34
services/settings/remote-settings.sys.mjs
View File

@@ -103,6 +103,7 @@ export async function jexlFilterFunc(entry, environment, collectionName) {
function remoteSettingsFunction() {
const _clients = new Map();
let _invalidatePolling = false;
let _initialized = false;
// If not explicitly specified, use the default signer.
const defaultOptions = {
@@ -304,6 +305,39 @@ function remoteSettingsFunction() {
trigger = "manual",
full = false,
} = {}) => {
if (AppConstants.MOZ_APP_VERSION) {
// Called multiple times on GeckoView due to bug 1730026
if (_initialized) {
return;
}
_initialized = true;
let importedFromDump = false;
for (const client of _clients.values()) {
const hasLocalDump = await lazy.Utils.hasLocalDump(
client.bucketName,
client.collectionName
);
if (hasLocalDump) {
const lastModified = await client.getLastModified();
const lastModifiedDump = await lazy.Utils.getLocalDumpLastModified(
client.bucketName,
client.collectionName
);
if (lastModified < lastModifiedDump) {
await client.maybeSync(lastModifiedDump, {
loadDump: true,
trigger,
});
importedFromDump = true;
}
}
}
if (importedFromDump) {
Services.obs.notifyObservers(null, "remote-settings:changes-poll-end");
}
return;
}
if (lazy.Utils.shouldSkipRemoteActivityDueToTests) {
return;
}