diff --git a/services/settings/Database.jsm b/services/settings/Database.jsm index b000ea90430f..11dba978544f 100644 --- a/services/settings/Database.jsm +++ b/services/settings/Database.jsm @@ -26,6 +26,10 @@ var EXPORTED_SYMBOLS = ["Database"]; * (with the objective of getting rid of kinto-offline-client) */ class Database { + static destroy() { + return destroyIDB(); + } + constructor(identifier) { ensureShutdownBlocker(); this.identifier = identifier; @@ -405,6 +409,35 @@ async function executeIDB(storeNames, callback, options = {}) { return promise.finally(finishedFn); } +async function destroyIDB() { + if (gDB) { + if (gShutdownStarted || Services.startup.shuttingDown) { + throw new lazy.IDBHelpers.ShutdownError( + "The application is shutting down", + "destroyIDB()" + ); + } + + // This will return immediately; the actual close will happen once + // there are no more running transactions. + gDB.close(); + const allTransactions = new Set([ + ...gPendingWriteOperations, + ...gPendingReadOnlyTransactions, + ]); + for (let transaction of Array.from(allTransactions)) { + try { + transaction.abort(); + } catch (ex) { + // Ignore errors to abort transactions, we'll destroy everything. + } + } + } + gDB = null; + gDBPromise = null; + return lazy.IDBHelpers.destroyIDB(); +} + function makeNestedObjectFromArr(arr, val, nestedFiltersObj) { const last = arr.length - 1; return arr.reduce((acc, cv, i) => { @@ -449,7 +482,7 @@ Database._shutdownHandler = () => { // Ensure we don't throw/break, because either way we're in shutdown. // In particular, `transaction.abort` can throw if the transaction - // is complete, ie if we manage to get called inbetween the + // is complete, ie if we manage to get called in between the // transaction completing, and our completion handler being called // to remove the item from the set. We don't care about that. if (ex.result != NS_ERROR_DOM_INDEXEDDB_NOT_ALLOWED_ERR) { diff --git a/services/settings/IDBHelpers.jsm b/services/settings/IDBHelpers.jsm index 5dc59c3687ef..1199d87e943c 100644 --- a/services/settings/IDBHelpers.jsm +++ b/services/settings/IDBHelpers.jsm @@ -202,10 +202,19 @@ async function openIDB(allowUpgrades = true) { }); } +function destroyIDB() { + const request = indexedDB.deleteDatabase(DB_NAME); + return new Promise((resolve, reject) => { + request.onerror = event => reject(new IndexedDBError(event.target.error)); + request.onsuccess = () => resolve(); + }); +} + var IDBHelpers = { bulkOperationHelper, executeIDB, openIDB, + destroyIDB, IndexedDBError, ShutdownError, }; diff --git a/services/settings/SyncHistory.jsm b/services/settings/SyncHistory.jsm index 46a43c76d7bf..44b911630f25 100644 --- a/services/settings/SyncHistory.jsm +++ b/services/settings/SyncHistory.jsm @@ -93,6 +93,14 @@ class SyncHistory { return entries; } + /** + * Return the most recent entry. + */ + async last() { + // List is sorted from newer to older. + return (await this.list())[0]; + } + /** * Wipe out the **whole** store. */ diff --git a/services/settings/remote-settings.js b/services/settings/remote-settings.js index c795901ffed9..350714a9ed2f 100644 --- a/services/settings/remote-settings.js +++ b/services/settings/remote-settings.js @@ -27,6 +27,7 @@ XPCOMUtils.defineLazyModuleGetters(lazy, { pushBroadcastService: "resource://gre/modules/PushBroadcastService.jsm", RemoteSettingsClient: "resource://services-settings/RemoteSettingsClient.jsm", SyncHistory: "resource://services-settings/SyncHistory.jsm", + Database: "resource://services-settings/Database.jsm", Utils: "resource://services-settings/Utils.jsm", FilterExpressions: "resource://gre/modules/components-utils/FilterExpressions.jsm", @@ -73,6 +74,13 @@ XPCOMUtils.defineLazyPreferenceGetter( 10 ); +XPCOMUtils.defineLazyPreferenceGetter( + lazy, + "gPrefDestroyBrokenEnabled", + PREF_SETTINGS_BRANCH + "destroy_broken_db_enabled", + true +); + /** * Default entry filtering function, in charge of excluding remote settings entries * where the JEXL expression evaluates into a falsy value. @@ -251,6 +259,38 @@ function remoteSettingsFunction() { } } + // When triggered from the daily timer, we try to recover a broken + // sync state by destroying the local DB completely and retrying from scratch. + if ( + lazy.gPrefDestroyBrokenEnabled && + trigger == "timer" && + (await isSynchronizationBroken()) + ) { + // We don't want to destroy the local DB if the failures are related to + // network or server errors though. + const lastStatus = await lazy.gSyncHistory.last(); + const lastErrorClass = + lazy.RemoteSettingsClient[lastStatus?.infos?.errorName] || Error; + const isLocalError = !( + lastErrorClass.prototype instanceof lazy.RemoteSettingsClient.APIError + ); + if (isLocalError) { + console.warn( + "Synchronization has failed consistently. Destroy database." + ); + // Clear the last ETag to refetch everything. + lazy.gPrefs.clearUserPref(PREF_SETTINGS_LAST_ETAG); + // Clear the history, to avoid re-destroying several times in a row. + await lazy.gSyncHistory.clear().catch(error => Cu.reportError(error)); + // Delete the whole IndexedDB database. + await lazy.Database.destroy().catch(error => Cu.reportError(error)); + } else { + console.warn( + `Synchronization is broken, but last error is ${lastStatus}` + ); + } + } + lazy.console.info("Start polling for changes"); Services.obs.notifyObservers( null, diff --git a/services/settings/test/unit/test_remote_settings_recover_broken.js b/services/settings/test/unit/test_remote_settings_recover_broken.js new file mode 100644 index 000000000000..f9e24455fa67 --- /dev/null +++ b/services/settings/test/unit/test_remote_settings_recover_broken.js @@ -0,0 +1,156 @@ +/* import-globals-from ../../../common/tests/unit/head_helpers.js */ + +const { AppConstants } = ChromeUtils.import( + "resource://gre/modules/AppConstants.jsm" +); +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm"); + +const { SyncHistory } = ChromeUtils.import( + "resource://services-settings/SyncHistory.jsm" +); +const { RemoteSettingsClient } = ChromeUtils.import( + "resource://services-settings/RemoteSettingsClient.jsm" +); +const { RemoteSettings } = ChromeUtils.import( + "resource://services-settings/remote-settings.js" +); +const { Utils } = ChromeUtils.import("resource://services-settings/Utils.jsm"); + +const PREF_SETTINGS_SERVER = "services.settings.server"; +const CHANGES_PATH = "/v1" + Utils.CHANGES_PATH; +const BROKEN_SYNC_THRESHOLD = 10; // See default pref value + +let server; +let client; +let maybeSyncBackup; + +async function clear_state() { + // Disable logging output. + Services.prefs.setCharPref("services.settings.loglevel", "critical"); + // Pull data from the test server. + Services.prefs.setCharPref( + PREF_SETTINGS_SERVER, + `http://localhost:${server.identity.primaryPort}/v1` + ); + + // Clear sync history. + await new SyncHistory("").clear(); + + // Simulate a response whose ETag gets incremented on each call + // (in order to generate several history entries, indexed by timestamp). + let timestamp = 1337; + server.registerPathHandler(CHANGES_PATH, (request, response) => { + response.setStatusLine(null, 200, "OK"); + response.setHeader("Content-Type", "application/json; charset=UTF-8"); + response.setHeader("Date", new Date(1000000).toUTCString()); + response.setHeader("ETag", `"${timestamp}"`); + response.write( + JSON.stringify({ + timestamp, + changes: [ + { + last_modified: ++timestamp, + bucket: "main", + collection: "desktop-manager", + }, + ], + }) + ); + }); + + // Restore original maybeSync() method between each test. + client.maybeSync = maybeSyncBackup; +} + +function run_test() { + // Set up an HTTP Server + server = new HttpServer(); + server.start(-1); + + client = RemoteSettings("desktop-manager"); + maybeSyncBackup = client.maybeSync; + + run_next_test(); + + registerCleanupFunction(() => { + server.stop(() => {}); + // Restore original maybeSync() method when test suite is done. + client.maybeSync = maybeSyncBackup; + }); +} + +add_task(clear_state); + +add_task(async function test_db_is_destroyed_when_sync_is_broken() { + // Simulate a successful sync. + client.maybeSync = async () => { + // Store some data in local DB. + await client.db.importChanges({}, 1515, []); + }; + await RemoteSettings.pollChanges({ trigger: "timer" }); + + // Register a client with a failing sync method. + client.maybeSync = () => { + throw new RemoteSettingsClient.InvalidSignatureError( + "main/desktop-manager" + ); + }; + + // Now obtain several failures in a row. + for (var i = 0; i < BROKEN_SYNC_THRESHOLD; i++) { + try { + await RemoteSettings.pollChanges({ trigger: "timer" }); + } catch (e) {} + } + + // Synchronization is in broken state. + Assert.equal( + await client.db.getLastModified(), + 1515, + "Local DB was not destroyed yet" + ); + + // Synchronize again. Broken state will be detected. + try { + await RemoteSettings.pollChanges({ trigger: "timer" }); + } catch (e) {} + + // DB was destroyed. + Assert.equal( + await client.db.getLastModified(), + null, + "Local DB was destroyed" + ); +}); + +add_task(clear_state); + +add_task(async function test_db_is_not_destroyed_when_state_is_server_error() { + // Since we don't mock the server endpoints to obtain the changeset of this + // collection, the call to `maybeSync()` will fail with network errors. + + // Store some data in local DB. + await client.db.importChanges({}, 1515, []); + + // Now obtain several failures in a row. + let lastError; + for (var i = 0; i < BROKEN_SYNC_THRESHOLD + 1; i++) { + try { + await RemoteSettings.pollChanges({ trigger: "timer" }); + } catch (e) { + lastError = e; + } + } + Assert.ok( + /Cannot parse server content/.test(lastError.message), + "Error is about server" + ); + // DB was not destroyed. + Assert.equal( + await client.db.getLastModified(), + 1515, + "Local DB was not destroyed" + ); +}); + +add_task(clear_state); diff --git a/services/settings/test/unit/xpcshell.ini b/services/settings/test/unit/xpcshell.ini index 523d8a7776f8..1e5df0a2485b 100644 --- a/services/settings/test/unit/xpcshell.ini +++ b/services/settings/test/unit/xpcshell.ini @@ -12,6 +12,7 @@ support-files = test_attachments_downloader/** [test_remote_settings_dump_lastmodified.js] [test_remote_settings_offline.js] [test_remote_settings_poll.js] +[test_remote_settings_recover_broken.js] [test_remote_settings_worker.js] [test_remote_settings_jexl_filters.js] [test_remote_settings_release_prefs.js]