Bug 1869060 - Add SQLite Online Backup API support via mozIStorageAsyncConnection. r=mak

Differential Revision: https://phabricator.services.mozilla.com/D195934

dom/cache/Connection.cpp

@@ -281,4 +281,11 @@ uint32_t Connection::DecreaseTransactionNestingLevel(
   return mBase->DecreaseTransactionNestingLevel(aProofOfLock);
 }
 
+NS_IMETHODIMP
+Connection::BackupToFileAsync(nsIFile* aDestinationFile,
+                              mozIStorageCompletionCallback* aCallback) {
+  // async methods are not supported
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
 }  // namespace mozilla::dom::cache

storage/mozIStorageAsyncConnection.idl

@@ -132,7 +132,7 @@ interface mozIStorageAsyncConnection : nsISupports {
   [noscript] void spinningSynchronousClose();
 
   /**
-   * Clone a database and make the clone read only if needed.
+   * Clone a database connection and make the clone read only if needed.
    * SQL Functions and attached on-disk databases are applied to the new clone.
    *
    * @param aReadOnly
@@ -368,4 +368,38 @@ interface mozIStorageAsyncConnection : nsISupports {
    * @return previous registered handler.
    */
   mozIStorageProgressHandler removeProgressHandler();
+
+  /**
+   * Makes a copy of a database asynchronously. This method can do an online
+   * backup of the database file, even if there are open connections actively
+   * using it (as a normal file copy can only be made if no connections are
+   * open on the database).
+   *
+   * While the copy is in the process of being made, the destination file
+   * will have a .tmp extension appended to it. In the event of a crash
+   * during the copy, this .tmp file will continue to exist, but will be
+   * an unusable partial copy.
+   *
+   * Once the copy has been completed, this method will automatically remove
+   * the .tmp extension.
+   *
+   * @param aDestinationFile
+   *        The destination on the file system to write the database copy.
+   * @param aCallback
+   *        A callback that will be notified when the operation is complete,
+   *        with the following arguments:
+   *        - status: the status of the operation, use this to check if making
+   *                  the copy was successful.
+   *        - value: unused.
+   * @throws NS_ERROR_ABORT
+   *         If the application has begun the process of shutting down already.
+   * @throws NS_ERROR_NOT_INITIALIZED
+   *         If the connection has already started or completed closing.
+   * @throws NS_ERROR_NOT_AVAILABLE
+   *         If the database does not support asynchronous operations.
+   * @throws NS_ERROR_NOT_INITIALIZED
+   *         If the execution thread cannot be acquired.
+   */
+  void backupToFileAsync(in nsIFile aDestinationFile,
+                         in mozIStorageCompletionCallback aCallback);
 };

storage/mozStorageConnection.cpp

@@ -48,6 +48,7 @@
 #include "mozilla/Printf.h"
 #include "mozilla/ProfilerLabels.h"
 #include "mozilla/RefPtr.h"
+#include "nsComponentManagerUtils.h"
 #include "nsProxyRelease.h"
 #include "nsStringFwd.h"
 #include "nsURLHelper.h"
@@ -583,6 +584,199 @@ class AsyncVacuumEvent final : public Runnable {
   Atomic<nsresult> mStatus;
 };
 
+/**
+ * A runnable to perform an SQLite database backup when there may be one or more
+ * open connections on that database.
+ */
+class AsyncBackupDatabaseFile final : public Runnable, public nsITimerCallback {
+ public:
+  NS_DECL_ISUPPORTS_INHERITED
+
+  /**
+   * @param aConnection The connection to the database being backed up.
+   * @param aNativeConnection The native connection to the database being backed
+   *                          up.
+   * @param aDestinationFile The destination file for the created backup.
+   * @param aCallback A callback to trigger once the backup process has
+   *                  completed. The callback will be supplied with an nsresult
+   *                  indicating whether or not the backup was successfully
+   *                  created. This callback will be called on the
+   *                  mConnection->eventTargetOpenedOn thread.
+   * @throws
+   */
+  AsyncBackupDatabaseFile(Connection* aConnection, sqlite3* aNativeConnection,
+                          nsIFile* aDestinationFile,
+                          mozIStorageCompletionCallback* aCallback)
+      : Runnable("storage::AsyncBackupDatabaseFile"),
+        mConnection(aConnection),
+        mNativeConnection(aNativeConnection),
+        mDestinationFile(aDestinationFile),
+        mCallback(aCallback),
+        mBackupFile(nullptr),
+        mBackupHandle(nullptr) {
+    MOZ_ASSERT(NS_IsMainThread());
+  }
+
+  NS_IMETHOD Run() override {
+    MOZ_ASSERT(!NS_IsMainThread());
+
+    nsAutoString path;
+    nsresult rv = mDestinationFile->GetPath(path);
+    if (NS_FAILED(rv)) {
+      return Dispatch(rv, nullptr);
+    }
+    // Put a .tmp on the end of the destination while the backup is underway.
+    // This extension will be stripped off after the backup successfully
+    // completes.
+    path.AppendLiteral(".tmp");
+
+    int srv = ::sqlite3_open(NS_ConvertUTF16toUTF8(path).get(), &mBackupFile);
+    if (srv != SQLITE_OK) {
+      return Dispatch(NS_ERROR_FAILURE, nullptr);
+    }
+
+    static const char* mainDBName = "main";
+
+    mBackupHandle = ::sqlite3_backup_init(mBackupFile, mainDBName,
+                                          mNativeConnection, mainDBName);
+    if (!mBackupHandle) {
+      MOZ_ALWAYS_TRUE(::sqlite3_close(mBackupFile) == SQLITE_OK);
+      return Dispatch(NS_ERROR_FAILURE, nullptr);
+    }
+
+    return DoStep();
+  }
+
+  NS_IMETHOD
+  Notify(nsITimer* aTimer) override { return DoStep(); }
+
+ private:
+  nsresult DoStep() {
+#define DISPATCH_AND_RETURN_IF_FAILED(rv) \
+  if (NS_FAILED(rv)) {                    \
+    return Dispatch(rv, nullptr);         \
+  }
+
+    // This guard is used to close the backup database in the event of
+    // some failure throughout this process. We release the exit guard
+    // only if we complete the backup successfully, or defer to another
+    // later call to DoStep.
+    auto guard = MakeScopeExit([&]() {
+      MOZ_ALWAYS_TRUE(::sqlite3_close(mBackupFile) == SQLITE_OK);
+      mBackupFile = nullptr;
+    });
+
+    MOZ_ASSERT(!NS_IsMainThread());
+    nsAutoString originalPath;
+    nsresult rv = mDestinationFile->GetPath(originalPath);
+    DISPATCH_AND_RETURN_IF_FAILED(rv);
+
+    nsAutoString tempPath = originalPath;
+    tempPath.AppendLiteral(".tmp");
+
+    nsCOMPtr<nsIFile> file =
+        do_CreateInstance("@mozilla.org/file/local;1", &rv);
+    DISPATCH_AND_RETURN_IF_FAILED(rv);
+
+    rv = file->InitWithPath(tempPath);
+    DISPATCH_AND_RETURN_IF_FAILED(rv);
+
+    // The number of milliseconds to wait between each batch of copies.
+    static constexpr uint32_t STEP_DELAY_MS = 250;
+    // The number of pages to copy per step
+    static constexpr int COPY_PAGES = 5;
+
+    int srv = ::sqlite3_backup_step(mBackupHandle, COPY_PAGES);
+    if (srv == SQLITE_OK || srv == SQLITE_BUSY || srv == SQLITE_LOCKED) {
+      // We're continuing the backup later. Release the guard to avoid closing
+      // the database.
+      guard.release();
+      // Queue up the next step
+      return NS_NewTimerWithCallback(getter_AddRefs(mTimer), this,
+                                     STEP_DELAY_MS, nsITimer::TYPE_ONE_SHOT,
+                                     GetCurrentSerialEventTarget());
+    }
+#ifdef DEBUG
+    if (srv != SQLITE_DONE) {
+      nsCString warnMsg;
+      warnMsg.AppendLiteral(
+          "The SQLite database copy could not be completed due to an error: ");
+      warnMsg.Append(::sqlite3_errmsg(mBackupFile));
+      NS_WARNING(warnMsg.get());
+    }
+#endif
+
+    (void)::sqlite3_backup_finish(mBackupHandle);
+    MOZ_ALWAYS_TRUE(::sqlite3_close(mBackupFile) == SQLITE_OK);
+    mBackupFile = nullptr;
+
+    // The database is already closed, so we can release this guard now.
+    guard.release();
+
+    if (srv != SQLITE_DONE) {
+      NS_WARNING("Failed to create database copy.");
+
+      // The partially created database file is not useful. Let's remove it.
+      rv = file->Remove(false);
+      if (NS_FAILED(rv)) {
+        NS_WARNING(
+            "Removing a partially backed up SQLite database file failed.");
+      }
+
+      return Dispatch(convertResultCode(srv), nullptr);
+    }
+
+    // Now that we've successfully created the copy, we'll strip off the .tmp
+    // extension.
+
+    nsAutoString leafName;
+    rv = mDestinationFile->GetLeafName(leafName);
+    DISPATCH_AND_RETURN_IF_FAILED(rv);
+
+    rv = file->RenameTo(nullptr, leafName);
+    DISPATCH_AND_RETURN_IF_FAILED(rv);
+
+#undef DISPATCH_AND_RETURN_IF_FAILED
+    return Dispatch(NS_OK, nullptr);
+  }
+
+  nsresult Dispatch(nsresult aResult, nsISupports* aValue) {
+    RefPtr<CallbackComplete> event =
+        new CallbackComplete(aResult, aValue, mCallback.forget());
+    return mConnection->eventTargetOpenedOn->Dispatch(event,
+                                                      NS_DISPATCH_NORMAL);
+  }
+
+  ~AsyncBackupDatabaseFile() override {
+    nsresult rv;
+    nsCOMPtr<nsIThread> thread =
+        do_QueryInterface(mConnection->eventTargetOpenedOn, &rv);
+    MOZ_ASSERT(NS_SUCCEEDED(rv));
+
+    // Handle ambiguous nsISupports inheritance.
+    NS_ProxyRelease("AsyncBackupDatabaseFile::mConnection", thread,
+                    mConnection.forget());
+    NS_ProxyRelease("AsyncBackupDatabaseFile::mDestinationFile", thread,
+                    mDestinationFile.forget());
+
+    // Generally, the callback will be released by CallbackComplete.
+    // However, if for some reason Run() is not executed, we still
+    // need to ensure that it is released here.
+    NS_ProxyRelease("AsyncInitializeClone::mCallback", thread,
+                    mCallback.forget());
+  }
+
+  RefPtr<Connection> mConnection;
+  sqlite3* mNativeConnection;
+  nsCOMPtr<nsITimer> mTimer;
+  nsCOMPtr<nsIFile> mDestinationFile;
+  nsCOMPtr<mozIStorageCompletionCallback> mCallback;
+  sqlite3* mBackupFile;
+  sqlite3_backup* mBackupHandle;
+};
+
+NS_IMPL_ISUPPORTS_INHERITED(AsyncBackupDatabaseFile, Runnable, nsITimerCallback)
+
 }  // namespace
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -2753,4 +2947,35 @@ uint32_t Connection::DecreaseTransactionNestingLevel(
   return --mTransactionNestingLevel;
 }
 
+NS_IMETHODIMP
+Connection::BackupToFileAsync(nsIFile* aDestinationFile,
+                              mozIStorageCompletionCallback* aCallback) {
+  NS_ENSURE_ARG(aDestinationFile);
+  NS_ENSURE_ARG(aCallback);
+  NS_ENSURE_TRUE(NS_IsMainThread(), NS_ERROR_NOT_SAME_THREAD);
+
+  // Abort if we're shutting down.
+  if (AppShutdown::IsInOrBeyond(ShutdownPhase::AppShutdownConfirmed)) {
+    return NS_ERROR_ABORT;
+  }
+  // Check if AsyncClose or Close were already invoked.
+  if (!connectionReady()) {
+    return NS_ERROR_NOT_INITIALIZED;
+  }
+  nsresult rv = ensureOperationSupported(ASYNCHRONOUS);
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+  nsIEventTarget* asyncThread = getAsyncExecutionTarget();
+  if (!asyncThread) {
+    return NS_ERROR_NOT_INITIALIZED;
+  }
+
+  // Create and dispatch our backup event to the execution thread.
+  nsCOMPtr<nsIRunnable> backupEvent =
+      new AsyncBackupDatabaseFile(this, mDBConn, aDestinationFile, aCallback);
+  rv = asyncThread->Dispatch(backupEvent, NS_DISPATCH_NORMAL);
+  return rv;
+}
+
 }  // namespace mozilla::storage

storage/test/unit/test_connection_online_backup.js

@@ -0,0 +1,211 @@
+/* Any copyright is dedicated to the Public Domain.
+http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+/**
+ * This test file tests the backupToFileAsync function on
+ * mozIStorageAsyncConnection, which is implemented for mozStorageConnection.
+ * (but not implemented for mozilla::dom::cache::Connection).
+ */
+
+// The name of the backup database file that will be created.
+const BACKUP_FILE_NAME = "test_storage.sqlite.backup";
+// The number of rows to insert into the test table in the source
+// database.
+const TEST_ROWS = 10;
+// The page size to set on the source database. During setup, we assert that
+// this does not match the default page size.
+const TEST_PAGE_SIZE = 512;
+
+/**
+ * This setup function creates a table inside of the test database and inserts
+ * some test rows. Critically, it keeps the connection to the database _open_
+ * so that we can test the scenario where a database is copied with existing
+ * open connections.
+ *
+ * The database is closed in a cleanup function.
+ */
+add_setup(async () => {
+  let conn = await openAsyncDatabase(getTestDB());
+
+  Assert.notEqual(
+    conn.defaultPageSize,
+    TEST_PAGE_SIZE,
+    "Should not default to having the TEST_PAGE_SIZE"
+  );
+
+  await executeSimpleSQLAsync(conn, "PRAGMA page_size = " + TEST_PAGE_SIZE);
+
+  let createStmt = conn.createAsyncStatement("CREATE TABLE test(name TEXT)");
+  await executeAsync(createStmt);
+  createStmt.finalize();
+
+  registerCleanupFunction(async () => {
+    await asyncClose(conn);
+  });
+});
+
+/**
+ * Erases the test table and inserts TEST_ROWS rows into it.
+ *
+ * @param {mozIStorageAsyncConnection} connection
+ *   The connection to use to prepare the database.
+ * @returns {Promise<undefined>}
+ */
+async function prepareSourceDatabase(connection) {
+  await executeSimpleSQLAsync(connection, "DELETE from test");
+  for (let i = 0; i < TEST_ROWS; ++i) {
+    let name = `Database row #${i}`;
+    let stmt = connection.createAsyncStatement(
+      "INSERT INTO test (name) VALUES (:name)"
+    );
+    stmt.params.name = name;
+    let result = await executeAsync(stmt);
+    stmt.finalize();
+    Assert.ok(Components.isSuccessCode(result), `Inserted test row #${i}`);
+  }
+}
+
+/**
+ * Gets the test DB prepared with the testing table and rows.
+ *
+ * @returns {Promise<mozIStorageAsyncConnection>}
+ */
+async function getPreparedAsyncDatabase() {
+  let connection = await openAsyncDatabase(getTestDB());
+  await prepareSourceDatabase(connection);
+  return connection;
+}
+
+/**
+ * Creates a copy of the database connected to via connection, and
+ * returns an nsIFile pointing at the created copy file once the
+ * copy is complete.
+ *
+ * @param {mozIStorageAsyncConnection} connection
+ *   A connection to a database that should be copied.
+ * @returns {Promise<nsIFile>}
+ */
+async function createCopy(connection) {
+  let destFilePath = PathUtils.join(PathUtils.profileDir, BACKUP_FILE_NAME);
+  let destFile = await IOUtils.getFile(destFilePath);
+  Assert.ok(
+    !(await IOUtils.exists(destFilePath)),
+    "Backup file shouldn't exist yet."
+  );
+
+  await new Promise(resolve => {
+    connection.backupToFileAsync(destFile, result => {
+      Assert.ok(Components.isSuccessCode(result));
+      resolve(result);
+    });
+  });
+
+  return destFile;
+}
+
+/**
+ * Opens up the database at file, asserts that the page_size matches
+ * TEST_PAGE_SIZE, and that the number of rows in the test table matches
+ * expectedEntries. Closes the connection after these assertions.
+ *
+ * @param {nsIFile} file
+ *   The database file to be opened and queried.
+ * @param {number} [expectedEntries=TEST_ROWS]
+ *   The expected number of rows in the test table. Defaults to TEST_ROWS.
+ * @returns {Promise<undefined>}
+ */
+async function assertSuccessfulCopy(file, expectedEntries = TEST_ROWS) {
+  let conn = await openAsyncDatabase(file);
+
+  await executeSimpleSQLAsync(conn, "PRAGMA page_size", resultSet => {
+    let result = resultSet.getNextRow();
+    Assert.equal(TEST_PAGE_SIZE, result.getResultByIndex(0).getAsUint32());
+  });
+
+  let stmt = conn.createAsyncStatement("SELECT COUNT(*) FROM test");
+  let results = await new Promise(resolve => {
+    executeAsync(stmt, resolve);
+  });
+  stmt.finalize();
+  let row = results.getNextRow();
+  let count = row.getResultByName("COUNT(*)");
+  Assert.equal(count, expectedEntries, "Got the expected entries");
+
+  Assert.ok(
+    !file.leafName.endsWith(".tmp"),
+    "Should not end in .tmp extension"
+  );
+
+  await asyncClose(conn);
+}
+
+/**
+ * Test the basic behaviour of backupToFileAsync, and ensure that the copied
+ * database has the same characteristics and contents as the source database.
+ */
+add_task(async function test_backupToFileAsync() {
+  let newConnection = await getPreparedAsyncDatabase();
+  let copyFile = await createCopy(newConnection);
+  Assert.ok(
+    await IOUtils.exists(copyFile.path),
+    "A new file was created by backupToFileAsync"
+  );
+
+  await assertSuccessfulCopy(copyFile);
+  await IOUtils.remove(copyFile.path);
+  await asyncClose(newConnection);
+});
+
+/**
+ * Tests that if insertions are underway during a copy, that those insertions
+ * show up in the copied database.
+ */
+add_task(async function test_backupToFileAsync_during_insert() {
+  let newConnection = await getPreparedAsyncDatabase();
+  const NEW_ENTRIES = 5;
+
+  let copyFilePromise = createCopy(newConnection);
+  let inserts = [];
+  for (let i = 0; i < NEW_ENTRIES; ++i) {
+    let name = `New database row #${i}`;
+    let stmt = newConnection.createAsyncStatement(
+      "INSERT INTO test (name) VALUES (:name)"
+    );
+    stmt.params.name = name;
+    inserts.push(executeAsync(stmt));
+    stmt.finalize();
+  }
+  await Promise.all(inserts);
+  let copyFile = await copyFilePromise;
+
+  Assert.ok(
+    await IOUtils.exists(copyFile.path),
+    "A new file was created by backupToFileAsync"
+  );
+
+  await assertSuccessfulCopy(copyFile, TEST_ROWS + NEW_ENTRIES);
+  await IOUtils.remove(copyFile.path);
+  await asyncClose(newConnection);
+});
+
+/**
+ * Tests the behaviour of backupToFileAsync as exposed through Sqlite.sys.mjs.
+ */
+add_task(async function test_backupToFileAsync_via_Sqlite_module() {
+  let xpcomConnection = await getPreparedAsyncDatabase();
+  let moduleConnection = await Sqlite.openConnection({
+    path: xpcomConnection.databaseFile.path,
+  });
+
+  let copyFilePath = PathUtils.join(PathUtils.profileDir, BACKUP_FILE_NAME);
+  await moduleConnection.backup(copyFilePath);
+  let copyFile = await IOUtils.getFile(copyFilePath);
+  Assert.ok(await IOUtils.exists(copyFilePath), "A new file was created");
+
+  await assertSuccessfulCopy(copyFile);
+  await IOUtils.remove(copyFile.path);
+  await moduleConnection.close();
+  await asyncClose(xpcomConnection);
+});

storage/test/unit/xpcshell.toml

@@ -36,6 +36,8 @@ skip-if = ["debug"]
 
 ["test_connection_interrupt.js"]
 
+["test_connection_online_backup.js"]
+
 ["test_default_journal_size_limit.js"]
 
 ["test_js_helpers.js"]

third_party/sqlite3/src/sqlite.symbols

@@ -5,6 +5,9 @@
 #include ../ext/lib.symbols
 sqlite3_aggregate_context
 sqlite3_auto_extension
+sqlite3_backup_finish
+sqlite3_backup_init
+sqlite3_backup_step
 sqlite3_bind_blob
 sqlite3_bind_double
 sqlite3_bind_int

toolkit/modules/Sqlite.sys.mjs

@@ -1163,6 +1163,33 @@ ConnectionData.prototype = Object.freeze({
       Cu.now() + Sqlite.TRANSACTIONS_TIMEOUT_MS * 0.2;
     return this._timeoutPromise;
   },
+
+  /**
+   * Asynchronously makes a copy of the SQLite database while there may still be
+   * open connections on it.
+   *
+   * @param {string} destFilePath
+   *   The path on the local filesystem to write the database copy. Any existing
+   *   file at this path will be overwritten.
+   * @return Promise<undefined, nsresult>
+   */
+  async backupToFile(destFilePath) {
+    if (!this._dbConn) {
+      return Promise.reject(
+        new Error("No opened database connection to create a backup from.")
+      );
+    }
+    let destFile = await IOUtils.getFile(destFilePath);
+    return new Promise((resolve, reject) => {
+      this._dbConn.backupToFileAsync(destFile, result => {
+        if (Components.isSuccessCode(result)) {
+          resolve();
+        } else {
+          reject(result);
+        }
+      });
+    });
+  },
 });
 
 /**
@@ -1955,6 +1982,19 @@ OpenedConnection.prototype = Object.freeze({
   interrupt() {
     this._connectionData.interrupt();
   },
+
+  /**
+   * Asynchronously makes a copy of the SQLite database while there may still be
+   * open connections on it.
+   *
+   * @param {string} destFilePath
+   *   The path on the local filesystem to write the database copy. Any existing
+   *   file at this path will be overwritten.
+   * @return Promise<undefined, nsresult>
+   */
+  backup(destFilePath) {
+    return this._connectionData.backupToFile(destFilePath);
+  },
 });
 
 export var Sqlite = {