Bug 669410 - Reduce the SQLite caches' default size in url-classifier. r=dcamp

browser/app/profile/firefox.js

@@ -734,19 +734,11 @@ pref("urlclassifier.gethashtables", "goog-phish-shavar,goog-malware-shavar");
 // the database.
 pref("urlclassifier.confirm-age", 2700);
 
-#ifdef MOZ_WIDGET_GTK2
-#define RESTRICT_CACHEMAX
-#endif
-#ifdef XP_OS2
-#define RESTRICT_CACHEMAX
-#endif
-
 // Maximum size of the sqlite3 cache during an update, in bytes
-#ifdef RESTRICT_CACHEMAX
+pref("urlclassifier.updatecachemax", 41943040);
-pref("urlclassifier.updatecachemax", 104857600);
+
-#else
+// Maximum size of the sqlite3 cache for lookups, in bytes
-pref("urlclassifier.updatecachemax", -1);
+pref("urlclassifier.lookupcachemax", 1048576);
-#endif
 
 // URL for checking the reason for a malware warning.
 pref("browser.safebrowsing.malware.reportURL", "http://safebrowsing.clients.google.com/safebrowsing/diagnostic?client=%NAME%&hl=%LOCALE%&site=");

toolkit/components/url-classifier/nsUrlClassifierDBService.cpp

@@ -162,6 +162,9 @@ static const PRLogModuleInfo *gUrlClassifierDbServiceLog = nsnull;
 #define UPDATE_CACHE_SIZE_PREF    "urlclassifier.updatecachemax"
 #define UPDATE_CACHE_SIZE_DEFAULT -1
 
+#define LOOKUP_CACHE_SIZE_PREF    "urlclassifier.lookupcachemax"
+#define LOOKUP_CACHE_SIZE_DEFAULT -1
+
 // Amount of time to spend updating before committing and delaying, in
 // seconds.  This is checked after each update stream, so the actual
 // time spent can be higher than this, depending on update stream size.
@@ -191,6 +194,7 @@ static PRBool gShuttingDownThread = PR_FALSE;
 static PRInt32 gFreshnessGuarantee = CONFIRM_AGE_DEFAULT_SEC;
 
 static PRInt32 gUpdateCacheSize = UPDATE_CACHE_SIZE_DEFAULT;
+static PRInt32 gLookupCacheSize = LOOKUP_CACHE_SIZE_DEFAULT;
 
 static PRInt32 gWorkingTimeThreshold = UPDATE_WORKING_TIME_DEFAULT;
 static PRInt32 gDelayTime = UPDATE_DELAY_TIME_DEFAULT;
@@ -1191,6 +1195,10 @@ private:
   // Construct a Prefix Tree with known prefixes
   nsresult ConstructPrefixTree();
 
+  // Set the SQLite cache size
+  nsresult SetCacheSize(mozIStorageConnection * aConnection,
+                        PRInt32 aCacheSize);
+
   nsCOMPtr<nsIFile> mDBFile;
 
   nsCOMPtr<nsICryptoHash> mCryptoHash;
@@ -3059,6 +3067,26 @@ nsUrlClassifierDBServiceWorker::FinishStream()
   return NS_OK;
 }
 
+nsresult
+nsUrlClassifierDBServiceWorker::SetCacheSize(
+  mozIStorageConnection * aConnection, PRInt32 aCacheSize)
+{
+  mozStorageStatementScoper scoper(mGetPageSizeStatement);
+  PRBool hasResult;
+  nsresult rv = mGetPageSizeStatement->ExecuteStep(&hasResult);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  NS_ASSERTION(hasResult, "Should always be able to get page size from sqlite");
+  PRUint32 pageSize = mGetPageSizeStatement->AsInt32(0);
+  PRUint32 cachePages = aCacheSize / pageSize;
+  nsCAutoString cacheSizePragma("PRAGMA cache_size=");
+  cacheSizePragma.AppendInt(cachePages);
+  rv = aConnection->ExecuteSimpleSQL(cacheSizePragma);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  return NS_OK;
+}
+
 nsresult
 nsUrlClassifierDBServiceWorker::SetupUpdate()
 {
@@ -3075,18 +3103,11 @@ nsUrlClassifierDBServiceWorker::SetupUpdate()
   NS_ENSURE_SUCCESS(rv, rv);
 
   if (gUpdateCacheSize > 0) {
-    PRBool hasResult;
+    rv = SetCacheSize(mConnection, gUpdateCacheSize);
-    rv = mGetPageSizeStatement->ExecuteStep(&hasResult);
     NS_ENSURE_SUCCESS(rv, rv);
-
+    if (gUpdateCacheSize != gLookupCacheSize) {
-    NS_ASSERTION(hasResult, "Should always be able to get page size from sqlite");
+      mGrewCache = PR_TRUE;
-    PRUint32 pageSize = mGetPageSizeStatement->AsInt32(0);
+    }
-    PRUint32 cachePages = gUpdateCacheSize / pageSize;
-    nsCAutoString cacheSizePragma("PRAGMA cache_size=");
-    cacheSizePragma.AppendInt(cachePages);
-    rv = mConnection->ExecuteSimpleSQL(cacheSizePragma);
-    NS_ENSURE_SUCCESS(rv, rv);
-    mGrewCache = PR_TRUE;
   }
 
   return NS_OK;
@@ -3337,6 +3358,14 @@ nsUrlClassifierDBServiceWorker::OpenDb()
   rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous=OFF"));
   NS_ENSURE_SUCCESS(rv, rv);
 
+  rv = connection->CreateStatement
+    (NS_LITERAL_CSTRING("PRAGMA page_size"),
+     getter_AddRefs(mGetPageSizeStatement));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  rv = SetCacheSize(connection, gLookupCacheSize);
+  NS_ENSURE_SUCCESS(rv, rv);
+
   if (newDB) {
     rv = connection->SetSchemaVersion(IMPLEMENTATION_VERSION);
     NS_ENSURE_SUCCESS(rv, rv);
@@ -3391,11 +3420,6 @@ nsUrlClassifierDBServiceWorker::OpenDb()
      getter_AddRefs(mInsertTableIdStatement));
   NS_ENSURE_SUCCESS(rv, rv);
 
-  rv = connection->CreateStatement
-    (NS_LITERAL_CSTRING("PRAGMA page_size"),
-     getter_AddRefs(mGetPageSizeStatement));
-  NS_ENSURE_SUCCESS(rv, rv);
-
   mConnection = connection;
 
   mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
@@ -3928,6 +3952,9 @@ nsUrlClassifierDBService::Init()
     rv = prefs->GetIntPref(UPDATE_CACHE_SIZE_PREF, &tmpint);
     PR_ATOMIC_SET(&gUpdateCacheSize, NS_SUCCEEDED(rv) ? tmpint : UPDATE_CACHE_SIZE_DEFAULT);
 
+    rv = prefs->GetIntPref(LOOKUP_CACHE_SIZE_PREF, &tmpint);
+    PR_ATOMIC_SET(&gLookupCacheSize, NS_SUCCEEDED(rv) ? tmpint : LOOKUP_CACHE_SIZE_DEFAULT);
+
     rv = prefs->GetIntPref(UPDATE_WORKING_TIME, &tmpint);
     PR_ATOMIC_SET(&gWorkingTimeThreshold,
                   NS_SUCCEEDED(rv) ? tmpint : UPDATE_WORKING_TIME_DEFAULT);
@@ -4234,6 +4261,10 @@ nsUrlClassifierDBService::Observe(nsISupports *aSubject, const char *aTopic,
       PRInt32 tmpint;
       rv = prefs->GetIntPref(UPDATE_CACHE_SIZE_PREF, &tmpint);
       PR_ATOMIC_SET(&gUpdateCacheSize, NS_SUCCEEDED(rv) ? tmpint : UPDATE_CACHE_SIZE_DEFAULT);
+    } else if (NS_LITERAL_STRING(LOOKUP_CACHE_SIZE_PREF).Equals(aData)) {
+      PRInt32 tmpint;
+      rv = prefs->GetIntPref(LOOKUP_CACHE_SIZE_PREF, &tmpint);
+      PR_ATOMIC_SET(&gLookupCacheSize, NS_SUCCEEDED(rv) ? tmpint : LOOKUP_CACHE_SIZE_DEFAULT);
     } else if (NS_LITERAL_STRING(UPDATE_WORKING_TIME).Equals(aData)) {
       PRInt32 tmpint;
       rv = prefs->GetIntPref(UPDATE_WORKING_TIME, &tmpint);

toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp

@@ -106,33 +106,42 @@ nsUrlClassifierPrefixSet::AddPrefixes(const PRUint32 * prefixes, PRUint32 aLengt
     return NS_OK;
   }
 
-  MutexAutoLock lock(mPrefixTreeLock);
+  nsTArray<PRUint32> mNewIndexPrefixes(mIndexPrefixes);
+  nsTArray<PRUint32> mNewIndexStarts(mIndexStarts);
+  nsTArray<PRUint16> mNewDeltas(mDeltas);
 
-  mIndexPrefixes.AppendElement(prefixes[0]);
+  mNewIndexPrefixes.AppendElement(prefixes[0]);
-  mIndexStarts.AppendElement(mDeltas.Length());
+  mNewIndexStarts.AppendElement(mNewDeltas.Length());
 
   PRUint32 numOfDeltas = 0;
   PRUint32 currentItem = prefixes[0];
   for (PRUint32 i = 1; i < aLength; i++) {
     if ((numOfDeltas >= DELTAS_LIMIT) ||
           (prefixes[i] - currentItem >= MAX_INDEX_DIFF)) {
-      mIndexStarts.AppendElement(mDeltas.Length());
+      mNewIndexStarts.AppendElement(mNewDeltas.Length());
-      mIndexPrefixes.AppendElement(prefixes[i]);
+      mNewIndexPrefixes.AppendElement(prefixes[i]);
       numOfDeltas = 0;
     } else {
       PRUint16 delta = prefixes[i] - currentItem;
-      mDeltas.AppendElement(delta);
+      mNewDeltas.AppendElement(delta);
       numOfDeltas++;
     }
     currentItem = prefixes[i];
   }
 
-  mIndexPrefixes.Compact();
+  mNewIndexPrefixes.Compact();
-  mIndexStarts.Compact();
+  mNewIndexStarts.Compact();
-  mDeltas.Compact();
+  mNewDeltas.Compact();
 
-  LOG(("Total number of indices: %d", mIndexPrefixes.Length()));
+  LOG(("Total number of indices: %d", mNewIndexPrefixes.Length()));
-  LOG(("Total number of deltas: %d", mDeltas.Length()));
+  LOG(("Total number of deltas: %d", mNewDeltas.Length()));
+
+  MutexAutoLock lock(mPrefixTreeLock);
+
+  // This just swaps some pointers
+  mIndexPrefixes.SwapElements(mNewIndexPrefixes);
+  mIndexStarts.SwapElements(mNewIndexStarts);
+  mDeltas.SwapElements(mNewDeltas);
 
   mHasPrefixes = PR_TRUE;
   mTreeIsReady.NotifyAll();
@@ -141,12 +150,12 @@ nsUrlClassifierPrefixSet::AddPrefixes(const PRUint32 * prefixes, PRUint32 aLengt
 }
 
 PRUint32 nsUrlClassifierPrefixSet::BinSearch(PRUint32 start,
-                                              PRUint32 end,
+                                             PRUint32 end,
-                                              PRUint32 target)
+                                             PRUint32 target)
 {
   while (start != end && end >= start) {
-    int i = start + ((end - start) >> 1);
+    PRUint32 i = start + ((end - start) >> 1);
-    int value = mIndexPrefixes[i];
+    PRUint32 value = mIndexPrefixes[i];
     if (value < target) {
       start = i + 1;
     } else if (value > target) {