corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1338970 - Introduce 2-step DB update scheme. (background update then swap in) r=francois,gcp

Browse Source 
MozReview-Commit-ID: 4C9N0tuSso0
This commit is contained in:
Henry Chang
2017-02-21 14:18:29 +08:00
parent 7266cc9962
commit c16ea9af44
2 changed files with 249 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

294
toolkit/components/url-classifier/Classifier.cpp
View File

@@ -34,6 +34,7 @@ extern mozilla::LazyLogModule gUrlClassifierDbServiceLog;
#define STORE_DIRECTORY NS_LITERAL_CSTRING("safebrowsing")
#define TO_DELETE_DIR_SUFFIX NS_LITERAL_CSTRING("-to_delete")
#define BACKUP_DIR_SUFFIX NS_LITERAL_CSTRING("-backup")
#define UPDATING_DIR_SUFFIX NS_LITERAL_CSTRING("-updating")
#define METADATA_SUFFIX NS_LITERAL_CSTRING(".metadata")
@@ -180,6 +181,13 @@ Classifier::SetupPathNames()
rv = mBackupDirectory->AppendNative(STORE_DIRECTORY + BACKUP_DIR_SUFFIX);
NS_ENSURE_SUCCESS(rv, rv);
// Directory where to be working on the update.
rv = mCacheDirectory->Clone(getter_AddRefs(mUpdatingDirectory));
NS_ENSURE_SUCCESS(rv, rv);
rv = mUpdatingDirectory->AppendNative(STORE_DIRECTORY + UPDATING_DIR_SUFFIX);
NS_ENSURE_SUCCESS(rv, rv);
// Directory where to move the backup so we can atomically
// delete (really move) it.
rv = mCacheDirectory->Clone(getter_AddRefs(mToDeleteDirectory));
@@ -225,9 +233,19 @@ Classifier::Open(nsIFile& aCacheDirectory)
NS_ENSURE_SUCCESS(rv, rv);
// Clean up any to-delete directories that haven't been deleted yet.
// This is still required for backward compatibility.
rv = CleanToDelete();
NS_ENSURE_SUCCESS(rv, rv);
// If we met a crash during the previous update, "safebrowsing-updating"
// directory will exist and let's remove it.
rv = mUpdatingDirectory->Remove(true);
if (NS_SUCCEEDED(rv)) {
// If the "safebrowsing-updating" exists, it implies a crash occurred
// in the previous update.
LOG(("We may have hit a crash in the previous update."));
}
// Check whether we have an incomplete update and recover from the
// backup if so.
rv = RecoverBackups();
@@ -260,6 +278,7 @@ Classifier::Reset()
mRootStoreDirectory->Remove(true);
mBackupDirectory->Remove(true);
mUpdatingDirectory->Remove(true);
mToDeleteDirectory->Remove(true);
CreateStoreDirectory();
@@ -347,11 +366,6 @@ Classifier::AbortUpdateAndReset(const nsCString& aTable)
// ResetTables will clear both in-memory & on-disk data.
ResetTables(Clear_All, nsTArray<nsCString> { aTable });
// Remove the backup and delete directory since we are aborting
// from an update.
Unused << RemoveBackupTables();
Unused << CleanToDelete();
}
void
@@ -527,9 +541,144 @@ Classifier::Check(const nsACString& aSpec,
return NS_OK;
}
static nsresult
SwapDirectoryContent(nsIFile* aDir1,
nsIFile* aDir2,
nsIFile* aParentDir,
nsIFile* aTempDir)
{
// Pre-condition: |aDir1| and |aDir2| are directory and their parent
// are both |aParentDir|.
//
// Post-condition: The locations where aDir1 and aDir2 point to will not
// change but their contents will be exchanged. If we failed
// to swap their content, everything will be rolled back.
nsAutoCString tempDirName;
aTempDir->GetNativeLeafName(tempDirName);
nsresult rv;
nsAutoCString dirName1, dirName2;
aDir1->GetNativeLeafName(dirName1);
aDir2->GetNativeLeafName(dirName2);
LOG(("Swapping directories %s and %s...", dirName1.get(),
dirName2.get()));
// 1. Rename "dirName1" to "temp"
rv = aDir1->RenameToNative(nullptr, tempDirName);
if (NS_FAILED(rv)) {
LOG(("Unable to rename %s to %s", dirName1.get(),
tempDirName.get()));
return rv; // Nothing to roll back.
}
// 1.1. Create a handle for temp directory. This is required since
// |nsIFile.rename| will not change the location where the
// object points to.
nsCOMPtr<nsIFile> tempDirectory;
rv = aParentDir->Clone(getter_AddRefs(tempDirectory));
rv = tempDirectory->AppendNative(tempDirName);
// 2. Rename "dirName2" to "dirName1".
rv = aDir2->RenameToNative(nullptr, dirName1);
if (NS_FAILED(rv)) {
LOG(("Failed to rename %s to %s. Rename temp directory back to %s",
dirName2.get(), dirName1.get(), dirName1.get()));
nsresult rbrv = tempDirectory->RenameToNative(nullptr, dirName1);
NS_ENSURE_SUCCESS(rbrv, rbrv);
return rv;
}
// 3. Rename "temp" to "dirName2".
rv = tempDirectory->RenameToNative(nullptr, dirName2);
if (NS_FAILED(rv)) {
LOG(("Failed to rename temp directory to %s. ", dirName2.get()));
// We've done (1) renaming "dir1 to temp" and
// (2) renaming "dir2 to dir1"
// so the rollback is
// (1) renaming "dir1 to dir2" and
// (2) renaming "temp to dir1"
nsresult rbrv; // rollback result
rbrv = aDir1->RenameToNative(nullptr, dirName2);
NS_ENSURE_SUCCESS(rbrv, rbrv);
rbrv = tempDirectory->RenameToNative(nullptr, dirName1);
NS_ENSURE_SUCCESS(rbrv, rbrv);
return rv;
}
return rv;
}
void
Classifier::RemoveUpdateIntermediaries()
{
// Remove old LookupCaches.
for (auto c: mNewLookupCaches) {
delete c;
}
mNewLookupCaches.Clear();
// Remove the "old" directory. (despite its looking-new name)
if (NS_FAILED(mUpdatingDirectory->Remove(true))) {
// If the directory is locked from removal for some reason,
// we will fail here and it doesn't matter until the next
// update. (the next udpate will fail due to the removable
// "safebrowsing-udpating" directory.)
LOG(("Failed to remove updating directory."));
}
}
nsresult
Classifier::SwapInNewTablesAndCleanup()
{
nsresult rv;
// Step 1. Swap in on-disk tables. The idea of using "safebrowsing-backup"
// as the intermediary directory is we can get databases recovered if
// crash occurred in any step of the swap. (We will recover from
// "safebrowsing-backup" in OpenDb().)
rv = SwapDirectoryContent(mUpdatingDirectory, // contains new tables
mRootStoreDirectory, // contains old tables
mCacheDirectory, // common parent dir
mBackupDirectory); // intermediary dir for swap
if (NS_FAILED(rv)) {
LOG(("Failed to swap in on-disk tables."));
RemoveUpdateIntermediaries();
return rv;
}
// Step 2. Swap in in-memory tables and update root directroy handles.
mLookupCaches.SwapElements(mNewLookupCaches);
for (auto c: mLookupCaches) {
c->UpdateRootDirHandle(mRootStoreDirectory);
}
// Step 3. Re-generate active tables based on on-disk tables.
rv = RegenActiveTables();
if (NS_FAILED(rv)) {
LOG(("Failed to re-generate active tables!"));
}
// Step 4. Clean up intermediaries for update.
RemoveUpdateIntermediaries();
// Step 5. Invalidate cached tableRequest request.
mIsTableRequestResultOutdated = true;
LOG(("Done swap in updated tables."));
return rv;
}
nsresult
Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates)
{
// Will run on the update thread after Bug 1339760.
// No lock is required except for CopyInUseDirForUpdate() since
// the table lookup may lead to database removal.
if (!aUpdates || aUpdates->Length() == 0) {
return NS_OK;
}
@@ -554,10 +703,18 @@ Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates)
{
ScopedUpdatesClearer scopedUpdatesClearer(aUpdates);
LOG(("Backup before update."));
rv = BackupTables();
NS_ENSURE_SUCCESS(rv, rv);
{
// TODO: Bug 1339050. This section should be mutual exclusive to
// LookupCache::Open() and HashStore::Open() since those operations
// might fail and cause the database to be removed. It'll be fine
// until we move "apply update" code off the worker thread.
rv = CopyInUseDirForUpdate(); // i.e. mUpdatingDirectory will be setup.
if (NS_FAILED(rv)) {
LOG(("Failed to copy in-use directory for update."));
return rv;
}
CopyInUseLookupCacheForUpdate(); // i.e. mNewLookupCaches will be setup.
}
LOG(("Applying %" PRIuSIZE " table updates.", aUpdates->Length()));
@@ -567,23 +724,22 @@ Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates)
// Run all updates for one table
nsCString updateTable(aUpdates->ElementAt(i)->TableName());
// Will update the mirrored in-memory and on-disk databases.
if (TableUpdate::Cast<TableUpdateV2>((*aUpdates)[i])) {
rv = UpdateHashStore(aUpdates, updateTable);
} else {
rv = UpdateTableV4(aUpdates, updateTable);
}
// We mark the table associated info outdated no matter the
// update is successful to avoid any possibile non-atomic update.
mIsTableRequestResultOutdated = true;
if (NS_FAILED(rv)) {
if (rv != NS_ERROR_OUT_OF_MEMORY) {
#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
DumpFailedUpdate();
#endif
// TODO: Bug 1339050 - Should be run on worker thread.
AbortUpdateAndReset(updateTable);
}
RemoveUpdateIntermediaries();
return rv;
}
}
@@ -591,21 +747,10 @@ Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates)
} // End of scopedUpdatesClearer scope.
rv = RegenActiveTables();
NS_ENSURE_SUCCESS(rv, rv);
LOG(("Cleaning up backups."));
// Move the backup directory away (signaling the transaction finished
// successfully). This is atomic.
rv = RemoveBackupTables();
NS_ENSURE_SUCCESS(rv, rv);
// Do the actual deletion of the backup files.
rv = CleanToDelete();
NS_ENSURE_SUCCESS(rv, rv);
LOG(("Done applying updates."));
rv = SwapInNewTablesAndCleanup();
if (NS_FAILED(rv)) {
LOG(("Failed to swap in new tables."));
}
if (LOG_ENABLED()) {
PRIntervalTime clockEnd = PR_IntervalNow();
@@ -613,7 +758,7 @@ Classifier::ApplyUpdates(nsTArray<TableUpdate*>* aUpdates)
PR_IntervalToMilliseconds(clockEnd - clockStart)));
}
return NS_OK;
return rv;
}
nsresult
@@ -815,11 +960,11 @@ Classifier::DumpFailedUpdate()
nsresult rv = failedUpdatekDirectory->GetNativeLeafName(failedUpdatekDirName);
NS_ENSURE_SUCCESS(rv, rv);
// Move backup to a clean "failed update" directory.
nsCOMPtr<nsIFile> backupDirectory;
if (NS_FAILED(mBackupDirectory->Clone(getter_AddRefs(backupDirectory))) ||
NS_FAILED(backupDirectory->MoveToNative(nullptr, failedUpdatekDirName))) {
LOG(("Failed to move backup to the \"failed update\" directory %s",
// Copy the in-use directory to a clean "failed update" directory.
nsCOMPtr<nsIFile> inUseDirectory;
if (NS_FAILED(mRootStoreDirectory->Clone(getter_AddRefs(inUseDirectory))) ||
NS_FAILED(inUseDirectory->CopyToNative(nullptr, failedUpdatekDirName))) {
LOG(("Failed to move in-use to the \"failed update\" directory %s",
failedUpdatekDirName.get()));
return NS_ERROR_FAILURE;
}
@@ -830,26 +975,20 @@ Classifier::DumpFailedUpdate()
#endif // MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
nsresult
Classifier::BackupTables()
Classifier::CopyInUseDirForUpdate()
{
// We have to work in reverse here: first move the normal directory
// away to be the backup directory, then copy the files over
// to the normal directory. This ensures that if we crash the backup
// dir always has a valid, complete copy, instead of a partial one,
// because that's the one we will copy over the normal store dir.
LOG(("Copy in-use directory content for update."));
nsCString backupDirName;
nsresult rv = mBackupDirectory->GetNativeLeafName(backupDirName);
// We copy everything from in-use directory to a temporary directory
// for updating.
nsCString updatingDirName;
nsresult rv = mUpdatingDirectory->GetNativeLeafName(updatingDirName);
NS_ENSURE_SUCCESS(rv, rv);
nsCString storeDirName;
rv = mRootStoreDirectory->GetNativeLeafName(storeDirName);
NS_ENSURE_SUCCESS(rv, rv);
rv = mRootStoreDirectory->MoveToNative(nullptr, backupDirName);
NS_ENSURE_SUCCESS(rv, rv);
rv = mRootStoreDirectory->CopyToNative(nullptr, storeDirName);
// Remove the destination directory first (just in case) the do the copy.
mUpdatingDirectory->Remove(true);
rv = mRootStoreDirectory->CopyToNative(nullptr, updatingDirName);
NS_ENSURE_SUCCESS(rv, rv);
// We moved some things to new places, so move the handles around, too.
@@ -859,21 +998,19 @@ Classifier::BackupTables()
return NS_OK;
}
nsresult
Classifier::RemoveBackupTables()
void
Classifier::CopyInUseLookupCacheForUpdate()
{
nsCString toDeleteName;
nsresult rv = mToDeleteDirectory->GetNativeLeafName(toDeleteName);
NS_ENSURE_SUCCESS(rv, rv);
MOZ_ASSERT(mNewLookupCaches.IsEmpty(), "Update intermediaries is forgotten to "
"be removed in the previous update.");
rv = mBackupDirectory->MoveToNative(nullptr, toDeleteName);
NS_ENSURE_SUCCESS(rv, rv);
// mBackupDirectory now points to toDelete, fix that up.
rv = SetupPathNames();
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
// TODO: Bug 1341183 - Implement deep copy function or copy ctor.
// For now we just re-open every table we have opened.
// Another option is to NOT pre-open these LookupCaches and
// do a "merge" instead of "swap" after update.
for (auto c: mLookupCaches) {
Unused << GetLookupCacheForUpdate(c->TableName());
}
}
nsresult
@@ -962,7 +1099,7 @@ Classifier::UpdateHashStore(nsTArray<TableUpdate*>* aUpdates,
LOG(("Classifier::UpdateHashStore(%s)", PromiseFlatCString(aTable).get()));
HashStore store(aTable, GetProvider(aTable), mRootStoreDirectory);
HashStore store(aTable, GetProvider(aTable), mUpdatingDirectory);
if (!CheckValidUpdate(aUpdates, store.TableName())) {
return NS_OK;
@@ -975,7 +1112,7 @@ Classifier::UpdateHashStore(nsTArray<TableUpdate*>* aUpdates,
// Read the part of the store that is (only) in the cache
LookupCacheV2* lookupCache =
LookupCache::Cast<LookupCacheV2>(GetLookupCache(store.TableName()));
LookupCache::Cast<LookupCacheV2>(GetLookupCacheForUpdate(store.TableName()));
if (!lookupCache) {
return NS_ERROR_FAILURE;
}
@@ -1069,7 +1206,7 @@ Classifier::UpdateTableV4(nsTArray<TableUpdate*>* aUpdates,
}
LookupCacheV4* lookupCache =
LookupCache::Cast<LookupCacheV4>(GetLookupCache(aTable));
LookupCache::Cast<LookupCacheV4>(GetLookupCacheForUpdate(aTable));
if (!lookupCache) {
return NS_ERROR_FAILURE;
}
@@ -1175,11 +1312,22 @@ Classifier::UpdateCache(TableUpdate* aUpdate)
}
LookupCache *
Classifier::GetLookupCache(const nsACString& aTable)
Classifier::GetLookupCache(const nsACString& aTable, bool aForUpdate)
{
for (uint32_t i = 0; i < mLookupCaches.Length(); i++) {
if (mLookupCaches[i]->TableName().Equals(aTable)) {
return mLookupCaches[i];
if (aForUpdate) {
return GetLookupCacheFrom(aTable, mNewLookupCaches, mUpdatingDirectory);
}
return GetLookupCacheFrom(aTable, mLookupCaches, mRootStoreDirectory);
}
LookupCache *
Classifier::GetLookupCacheFrom(const nsACString& aTable,
nsTArray<LookupCache*>& aLookupCaches,
nsIFile* aRootStoreDirectory)
{
for (uint32_t i = 0; i < aLookupCaches.Length(); i++) {
if (aLookupCaches[i]->TableName().Equals(aTable)) {
return aLookupCaches[i];
}
}
@@ -1189,9 +1337,9 @@ Classifier::GetLookupCache(const nsACString& aTable)
UniquePtr<LookupCache> cache;
nsCString provider = GetProvider(aTable);
if (StringEndsWith(aTable, NS_LITERAL_CSTRING("-proto"))) {
cache = MakeUnique<LookupCacheV4>(aTable, provider, mRootStoreDirectory);
cache = MakeUnique<LookupCacheV4>(aTable, provider, aRootStoreDirectory);
} else {
cache = MakeUnique<LookupCacheV2>(aTable, provider, mRootStoreDirectory);
cache = MakeUnique<LookupCacheV2>(aTable, provider, aRootStoreDirectory);
}
nsresult rv = cache->Init();
@@ -1205,7 +1353,7 @@ Classifier::GetLookupCache(const nsACString& aTable)
}
return nullptr;
}
mLookupCaches.AppendElement(cache.get());
aLookupCaches.AppendElement(cache.get());
return cache.release();
}