corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 669410 - Reduce the SQLite caches' default size in url-classifier. r=dcamp

Browse Source
This commit is contained in:
Gian-Carlo Pascutto
2011-09-08 22:15:37 +02:00
parent 30d9f4fbfd
commit d4d482d799
3 changed files with 75 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
browser/app/profile/firefox.js
View File

@@ -734,19 +734,11 @@ pref("urlclassifier.gethashtables", "goog-phish-shavar,goog-malware-shavar");
// the database.
pref("urlclassifier.confirm-age", 2700);
#ifdef MOZ_WIDGET_GTK2
#define RESTRICT_CACHEMAX
#endif
#ifdef XP_OS2
#define RESTRICT_CACHEMAX
#endif
// Maximum size of the sqlite3 cache during an update, in bytes
#ifdef RESTRICT_CACHEMAX
pref("urlclassifier.updatecachemax", 104857600);
#else
pref("urlclassifier.updatecachemax", -1);
#endif
pref("urlclassifier.updatecachemax", 41943040);
// Maximum size of the sqlite3 cache for lookups, in bytes
pref("urlclassifier.lookupcachemax", 1048576);
// URL for checking the reason for a malware warning.
pref("browser.safebrowsing.malware.reportURL", "http://safebrowsing.clients.google.com/safebrowsing/diagnostic?client=%NAME%&hl=%LOCALE%&site=");

61
toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
View File

@@ -162,6 +162,9 @@ static const PRLogModuleInfo *gUrlClassifierDbServiceLog = nsnull;
#define UPDATE_CACHE_SIZE_PREF "urlclassifier.updatecachemax"
#define UPDATE_CACHE_SIZE_DEFAULT -1
#define LOOKUP_CACHE_SIZE_PREF "urlclassifier.lookupcachemax"
#define LOOKUP_CACHE_SIZE_DEFAULT -1
// Amount of time to spend updating before committing and delaying, in
// seconds. This is checked after each update stream, so the actual
// time spent can be higher than this, depending on update stream size.
@@ -191,6 +194,7 @@ static PRBool gShuttingDownThread = PR_FALSE;
static PRInt32 gFreshnessGuarantee = CONFIRM_AGE_DEFAULT_SEC;
static PRInt32 gUpdateCacheSize = UPDATE_CACHE_SIZE_DEFAULT;
static PRInt32 gLookupCacheSize = LOOKUP_CACHE_SIZE_DEFAULT;
static PRInt32 gWorkingTimeThreshold = UPDATE_WORKING_TIME_DEFAULT;
static PRInt32 gDelayTime = UPDATE_DELAY_TIME_DEFAULT;
@@ -1191,6 +1195,10 @@ private:
// Construct a Prefix Tree with known prefixes
nsresult ConstructPrefixTree();
// Set the SQLite cache size
nsresult SetCacheSize(mozIStorageConnection * aConnection,
PRInt32 aCacheSize);
nsCOMPtr<nsIFile> mDBFile;
nsCOMPtr<nsICryptoHash> mCryptoHash;
@@ -3059,6 +3067,26 @@ nsUrlClassifierDBServiceWorker::FinishStream()
return NS_OK;
}
nsresult
nsUrlClassifierDBServiceWorker::SetCacheSize(
mozIStorageConnection * aConnection, PRInt32 aCacheSize)
{
mozStorageStatementScoper scoper(mGetPageSizeStatement);
PRBool hasResult;
nsresult rv = mGetPageSizeStatement->ExecuteStep(&hasResult);
NS_ENSURE_SUCCESS(rv, rv);
NS_ASSERTION(hasResult, "Should always be able to get page size from sqlite");
PRUint32 pageSize = mGetPageSizeStatement->AsInt32(0);
PRUint32 cachePages = aCacheSize / pageSize;
nsCAutoString cacheSizePragma("PRAGMA cache_size=");
cacheSizePragma.AppendInt(cachePages);
rv = aConnection->ExecuteSimpleSQL(cacheSizePragma);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
nsresult
nsUrlClassifierDBServiceWorker::SetupUpdate()
{
@@ -3075,19 +3103,12 @@ nsUrlClassifierDBServiceWorker::SetupUpdate()
NS_ENSURE_SUCCESS(rv, rv);
if (gUpdateCacheSize > 0) {
PRBool hasResult;
rv = mGetPageSizeStatement->ExecuteStep(&hasResult);
NS_ENSURE_SUCCESS(rv, rv);
NS_ASSERTION(hasResult, "Should always be able to get page size from sqlite");
PRUint32 pageSize = mGetPageSizeStatement->AsInt32(0);
PRUint32 cachePages = gUpdateCacheSize / pageSize;
nsCAutoString cacheSizePragma("PRAGMA cache_size=");
cacheSizePragma.AppendInt(cachePages);
rv = mConnection->ExecuteSimpleSQL(cacheSizePragma);
rv = SetCacheSize(mConnection, gUpdateCacheSize);
NS_ENSURE_SUCCESS(rv, rv);
if (gUpdateCacheSize != gLookupCacheSize) {
mGrewCache = PR_TRUE;
}
}
return NS_OK;
}
@@ -3337,6 +3358,14 @@ nsUrlClassifierDBServiceWorker::OpenDb()
rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous=OFF"));
NS_ENSURE_SUCCESS(rv, rv);
rv = connection->CreateStatement
(NS_LITERAL_CSTRING("PRAGMA page_size"),
getter_AddRefs(mGetPageSizeStatement));
NS_ENSURE_SUCCESS(rv, rv);
rv = SetCacheSize(connection, gLookupCacheSize);
NS_ENSURE_SUCCESS(rv, rv);
if (newDB) {
rv = connection->SetSchemaVersion(IMPLEMENTATION_VERSION);
NS_ENSURE_SUCCESS(rv, rv);
@@ -3391,11 +3420,6 @@ nsUrlClassifierDBServiceWorker::OpenDb()
getter_AddRefs(mInsertTableIdStatement));
NS_ENSURE_SUCCESS(rv, rv);
rv = connection->CreateStatement
(NS_LITERAL_CSTRING("PRAGMA page_size"),
getter_AddRefs(mGetPageSizeStatement));
NS_ENSURE_SUCCESS(rv, rv);
mConnection = connection;
mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
@@ -3928,6 +3952,9 @@ nsUrlClassifierDBService::Init()
rv = prefs->GetIntPref(UPDATE_CACHE_SIZE_PREF, &tmpint);
PR_ATOMIC_SET(&gUpdateCacheSize, NS_SUCCEEDED(rv) ? tmpint : UPDATE_CACHE_SIZE_DEFAULT);
rv = prefs->GetIntPref(LOOKUP_CACHE_SIZE_PREF, &tmpint);
PR_ATOMIC_SET(&gLookupCacheSize, NS_SUCCEEDED(rv) ? tmpint : LOOKUP_CACHE_SIZE_DEFAULT);
rv = prefs->GetIntPref(UPDATE_WORKING_TIME, &tmpint);
PR_ATOMIC_SET(&gWorkingTimeThreshold,
NS_SUCCEEDED(rv) ? tmpint : UPDATE_WORKING_TIME_DEFAULT);
@@ -4234,6 +4261,10 @@ nsUrlClassifierDBService::Observe(nsISupports *aSubject, const char *aTopic,
PRInt32 tmpint;
rv = prefs->GetIntPref(UPDATE_CACHE_SIZE_PREF, &tmpint);
PR_ATOMIC_SET(&gUpdateCacheSize, NS_SUCCEEDED(rv) ? tmpint : UPDATE_CACHE_SIZE_DEFAULT);
} else if (NS_LITERAL_STRING(LOOKUP_CACHE_SIZE_PREF).Equals(aData)) {
PRInt32 tmpint;
rv = prefs->GetIntPref(LOOKUP_CACHE_SIZE_PREF, &tmpint);
PR_ATOMIC_SET(&gLookupCacheSize, NS_SUCCEEDED(rv) ? tmpint : LOOKUP_CACHE_SIZE_DEFAULT);
} else if (NS_LITERAL_STRING(UPDATE_WORKING_TIME).Equals(aData)) {
PRInt32 tmpint;
rv = prefs->GetIntPref(UPDATE_WORKING_TIME, &tmpint);

35
toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp
View File

@@ -106,33 +106,42 @@ nsUrlClassifierPrefixSet::AddPrefixes(const PRUint32 * prefixes, PRUint32 aLengt
return NS_OK;
}
MutexAutoLock lock(mPrefixTreeLock);
nsTArray<PRUint32> mNewIndexPrefixes(mIndexPrefixes);
nsTArray<PRUint32> mNewIndexStarts(mIndexStarts);
nsTArray<PRUint16> mNewDeltas(mDeltas);
mIndexPrefixes.AppendElement(prefixes[0]);
mIndexStarts.AppendElement(mDeltas.Length());
mNewIndexPrefixes.AppendElement(prefixes[0]);
mNewIndexStarts.AppendElement(mNewDeltas.Length());
PRUint32 numOfDeltas = 0;
PRUint32 currentItem = prefixes[0];
for (PRUint32 i = 1; i < aLength; i++) {
if ((numOfDeltas >= DELTAS_LIMIT) ||
(prefixes[i] - currentItem >= MAX_INDEX_DIFF)) {
mIndexStarts.AppendElement(mDeltas.Length());
mIndexPrefixes.AppendElement(prefixes[i]);
mNewIndexStarts.AppendElement(mNewDeltas.Length());
mNewIndexPrefixes.AppendElement(prefixes[i]);
numOfDeltas = 0;
} else {
PRUint16 delta = prefixes[i] - currentItem;
mDeltas.AppendElement(delta);
mNewDeltas.AppendElement(delta);
numOfDeltas++;
}
currentItem = prefixes[i];
}
mIndexPrefixes.Compact();
mIndexStarts.Compact();
mDeltas.Compact();
mNewIndexPrefixes.Compact();
mNewIndexStarts.Compact();
mNewDeltas.Compact();
LOG(("Total number of indices: %d", mIndexPrefixes.Length()));
LOG(("Total number of deltas: %d", mDeltas.Length()));
LOG(("Total number of indices: %d", mNewIndexPrefixes.Length()));
LOG(("Total number of deltas: %d", mNewDeltas.Length()));
MutexAutoLock lock(mPrefixTreeLock);
// This just swaps some pointers
mIndexPrefixes.SwapElements(mNewIndexPrefixes);
mIndexStarts.SwapElements(mNewIndexStarts);
mDeltas.SwapElements(mNewDeltas);
mHasPrefixes = PR_TRUE;
mTreeIsReady.NotifyAll();
@@ -145,8 +154,8 @@ PRUint32 nsUrlClassifierPrefixSet::BinSearch(PRUint32 start,
PRUint32 target)
{
while (start != end && end >= start) {
int i = start + ((end - start) >> 1);
int value = mIndexPrefixes[i];
PRUint32 i = start + ((end - start) >> 1);
PRUint32 value = mIndexPrefixes[i];
if (value < target) {
start = i + 1;
} else if (value > target) {