corysanin/tubestation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug 1333328 - Refactor cache miss handling mechanism for V2. r=francois

Browse Source 
In this patch, we will make Safebrowsing V2 caching use the same algorithm as V4.
So we remove "mMissCache" for negative caching and TableFresness check for
positive caching.

But Safebrowsing V2 doesn't contain negative/positive cache duration information in
gethash response. So we hard-code a fixed value, 15 minutes, as cache duration.
In this way, we can sync the mechanism we handle caching for V2 and V4.

An extra effort for V2 here is that we need to manually record prefixes misses
because we won't get any response for those prefixes(implemented in
nsUrlClassifierLookupCallback::CacheMisses).
This commit is contained in:
DimiL
2017-05-04 09:38:14 +08:00
parent f55a863750
commit a0b8501692
23 changed files with 754 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
toolkit/components/url-classifier/tests/gtest/Common.cpp
View File

@@ -137,7 +137,6 @@ PrefixArrayToAddPrefixArrayV2(const nsTArray<nsCString>& prefixArray,
Prefix hash;
static_assert(sizeof(hash.buf) == PREFIX_SIZE, "Prefix must be 4 bytes length");
memcpy(hash.buf, prefixArray[i].BeginReading(), PREFIX_SIZE);
MOZ_ASSERT(prefixArray[i].Length() == PREFIX_SIZE);
AddPrefix *add = out.AppendElement(fallible);
if (!add) {
@@ -163,21 +162,42 @@ GeneratePrefix(const nsCString& aFragment, uint8_t aLength)
return hash;
}
UniquePtr<LookupCacheV4>
SetupLookupCacheV4(const _PrefixArray& prefixArray)
static nsresult
BuildCache(LookupCacheV2* cache, const _PrefixArray& prefixArray)
{
AddPrefixArray prefixes;
AddCompleteArray completions;
nsresult rv = PrefixArrayToAddPrefixArrayV2(prefixArray, prefixes);
if (NS_FAILED(rv)) {
return rv;
}
EntrySort(prefixes);
return cache->Build(prefixes, completions);
}
static nsresult
BuildCache(LookupCacheV4* cache, const _PrefixArray& prefixArray)
{
PrefixStringMap map;
PrefixArrayToPrefixStringMap(prefixArray, map);
return cache->Build(map);
}
template<typename T>
UniquePtr<T>
SetupLookupCache(const _PrefixArray& prefixArray)
{
nsCOMPtr<nsIFile> file;
NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, getter_AddRefs(file));
file->AppendNative(GTEST_SAFEBROWSING_DIR);
UniquePtr<LookupCacheV4> cache = MakeUnique<LookupCacheV4>(GTEST_TABLE, EmptyCString(), file);
UniquePtr<T> cache = MakeUnique<T>(GTEST_TABLE, EmptyCString(), file);
nsresult rv = cache->Init();
EXPECT_EQ(rv, NS_OK);
PrefixStringMap map;
PrefixArrayToPrefixStringMap(prefixArray, map);
rv = cache->Build(map);
rv = BuildCache(cache.get(), prefixArray);
EXPECT_EQ(rv, NS_OK);
return Move(cache);

3
toolkit/components/url-classifier/tests/gtest/Common.h
View File

@@ -45,4 +45,5 @@ nsresult PrefixArrayToAddPrefixArrayV2(const nsTArray<nsCString>& prefixArray,
nsCString GeneratePrefix(const nsCString& aFragment, uint8_t aLength);
// Create a LookupCacheV4 object with sepecified prefix array.
UniquePtr<LookupCacheV4> SetupLookupCacheV4(const _PrefixArray& prefixArray);
template<typename T>
UniquePtr<T> SetupLookupCache(const _PrefixArray& prefixArray);

149
toolkit/components/url-classifier/tests/gtest/TestCachingV4.cpp → toolkit/components/url-classifier/tests/gtest/TestCaching.cpp
View File

@@ -7,6 +7,32 @@
#define EXPIRED_TIME_SEC (PR_Now() / PR_USEC_PER_SEC - 3600)
#define NOTEXPIRED_TIME_SEC (PR_Now() / PR_USEC_PER_SEC + 3600)
static void
SetupCacheEntry(LookupCacheV2* aLookupCache,
const nsCString& aCompletion,
bool aNegExpired = false,
bool aPosExpired = false)
{
AddCompleteArray completes;
AddCompleteArray emptyCompletes;
MissPrefixArray misses;
MissPrefixArray emptyMisses;
nsCOMPtr<nsICryptoHash> cryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID);
AddComplete* add = completes.AppendElement(fallible);
add->complete.FromPlaintext(aCompletion, cryptoHash);
Prefix* prefix = misses.AppendElement(fallible);
prefix->FromPlaintext(aCompletion, cryptoHash);
int64_t negExpirySec = aNegExpired ? EXPIRED_TIME_SEC : NOTEXPIRED_TIME_SEC;
aLookupCache->AddGethashResultToCache(emptyCompletes, misses, negExpirySec);
int64_t posExpirySec = aPosExpired ? EXPIRED_TIME_SEC : NOTEXPIRED_TIME_SEC;
aLookupCache->AddGethashResultToCache(completes, emptyMisses, posExpirySec);
}
static void
SetupCacheEntry(LookupCacheV4* aLookupCache,
const nsCString& aCompletion,
@@ -28,19 +54,20 @@ SetupCacheEntry(LookupCacheV4* aLookupCache,
aLookupCache->AddFullHashResponseToCache(map);
}
template<typename T>
void
TestCache(const Completion aCompletion,
bool aExpectedHas,
bool aExpectedConfirmed,
bool aExpectedFromCache,
LookupCacheV4* aCache = nullptr)
bool aExpectedInCache,
T* aCache = nullptr)
{
bool has, fromCache, confirmed;
bool has, inCache, confirmed;
uint32_t matchLength;
TableFreshnessMap dummy;
if (aCache) {
aCache->Has(aCompletion, dummy, 0, &has, &matchLength, &confirmed, &fromCache);
aCache->Has(aCompletion, &has, &matchLength, &confirmed);
inCache = aCache->IsInCache(aCompletion.ToUint32());
} else {
_PrefixArray array = { GeneratePrefix(_Fragment("cache.notexpired.com/"), 10),
GeneratePrefix(_Fragment("cache.expired.com/"), 8),
@@ -48,65 +75,71 @@ TestCache(const Completion aCompletion,
GeneratePrefix(_Fragment("small.com/"), 4)
};
UniquePtr<LookupCacheV4> cache = SetupLookupCacheV4(array);
UniquePtr<T> cache = SetupLookupCache<T>(array);
// Create an expired entry and a non-expired entry
SetupCacheEntry(cache.get(), _Fragment("cache.notexpired.com/"));
SetupCacheEntry(cache.get(), _Fragment("cache.expired.com/"), true, true);
cache->Has(aCompletion, dummy, 0, &has, &matchLength, &confirmed, &fromCache);
cache->Has(aCompletion, &has, &matchLength, &confirmed);
inCache = cache->IsInCache(aCompletion.ToUint32());
}
EXPECT_EQ(has, aExpectedHas);
EXPECT_EQ(confirmed, aExpectedConfirmed);
EXPECT_EQ(fromCache, aExpectedFromCache);
EXPECT_EQ(inCache, aExpectedInCache);
}
template<typename T>
void
TestCache(const _Fragment& aFragment,
bool aExpectedHas,
bool aExpectedConfirmed,
bool aExpectedFromCache,
LookupCacheV4* aCache = nullptr)
bool aExpectedInCache,
T* aCache = nullptr)
{
Completion lookupHash;
nsCOMPtr<nsICryptoHash> cryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID);
lookupHash.FromPlaintext(aFragment, cryptoHash);
TestCache(lookupHash, aExpectedHas, aExpectedConfirmed, aExpectedFromCache, aCache);
TestCache<T>(lookupHash, aExpectedHas, aExpectedConfirmed, aExpectedInCache, aCache);
}
// This testcase check the returned result of |Has| API if fullhash cannot match
// any prefix in the local database.
TEST(CachingV4, NotFound)
TEST(UrlClassifierCaching, NotFound)
{
TestCache(_Fragment("nomatch.com/"), false, false, false);
TestCache<LookupCacheV2>(_Fragment("nomatch.com/"), false, false, false);
TestCache<LookupCacheV4>(_Fragment("nomatch.com/"), false, false, false);
}
// This testcase check the returned result of |Has| API if fullhash find a match
// in the local database but not in the cache.
TEST(CachingV4, NotInCache)
TEST(UrlClassifierCaching, NotInCache)
{
TestCache(_Fragment("gound.com/"), true, false, false);
TestCache<LookupCacheV2>(_Fragment("gound.com/"), true, false, false);
TestCache<LookupCacheV4>(_Fragment("gound.com/"), true, false, false);
}
// This testcase check the returned result of |Has| API if fullhash matches
// a cache entry in positive cache.
TEST(CachingV4, InPositiveCacheNotExpired)
TEST(UrlClassifierCaching, InPositiveCacheNotExpired)
{
TestCache(_Fragment("cache.notexpired.com/"), true, true, true);
TestCache<LookupCacheV2>(_Fragment("cache.notexpired.com/"), true, true, true);
TestCache<LookupCacheV4>(_Fragment("cache.notexpired.com/"), true, true, true);
}
// This testcase check the returned result of |Has| API if fullhash matches
// a cache entry in positive cache but that it is expired.
TEST(CachingV4, InPositiveCacheExpired)
TEST(UrlClassifierCaching, InPositiveCacheExpired)
{
TestCache(_Fragment("cache.expired.com/"), true, false, true);
TestCache<LookupCacheV2>(_Fragment("cache.expired.com/"), true, false, true);
TestCache<LookupCacheV4>(_Fragment("cache.expired.com/"), true, false, true);
}
// This testcase check the returned result of |Has| API if fullhash matches
// a cache entry in negative cache.
TEST(CachingV4, InNegativeCacheNotExpired)
TEST(UrlClassifierCaching, InNegativeCacheNotExpired)
{
// Create a fullhash whose prefix matches the prefix in negative cache
// but completion doesn't match any fullhash in positive cache.
@@ -124,12 +157,13 @@ TEST(CachingV4, InNegativeCacheNotExpired)
// it can match the prefix in database.
memcpy(fullhash.buf, prefix.buf, 10);
TestCache(fullhash, false, false, true);
TestCache<LookupCacheV2>(fullhash, false, false, true);
TestCache<LookupCacheV4>(fullhash, false, false, true);
}
// This testcase check the returned result of |Has| API if fullhash matches
// a cache entry in negative cache but that entry is expired.
TEST(CachingV4, InNegativeCacheExpired)
TEST(UrlClassifierCaching, InNegativeCacheExpired)
{
// Create a fullhash whose prefix is in the cache.
nsCOMPtr<nsICryptoHash> cryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID);
@@ -142,7 +176,8 @@ TEST(CachingV4, InNegativeCacheExpired)
memcpy(fullhash.buf, prefix.buf, 10);
TestCache(fullhash, true, false, true);
TestCache<LookupCacheV2>(fullhash, true, false, true);
TestCache<LookupCacheV4>(fullhash, true, false, true);
}
#define CACHED_URL _Fragment("cache.com/")
@@ -158,15 +193,15 @@ TEST(CachingV4, InNegativeCacheExpired)
// 4. an entry whose negative cache time and positive cache time are expired
// After calling |InvalidateExpiredCacheEntry| API, entries with expired
// negative time should be removed from cache(2 & 4)
TEST(CachingV4, InvalidateExpiredCacheEntry)
template<typename T>
void TestInvalidateExpiredCacheEntry()
{
_PrefixArray array = { GeneratePrefix(CACHED_URL, 10),
GeneratePrefix(NEG_CACHE_EXPIRED_URL, 8),
GeneratePrefix(POS_CACHE_EXPIRED_URL, 5),
GeneratePrefix(BOTH_CACHE_EXPIRED_URL, 4)
};
UniquePtr<LookupCacheV4> cache = SetupLookupCacheV4(array);
UniquePtr<T> cache = SetupLookupCache<T>(array);
SetupCacheEntry(cache.get(), CACHED_URL, false, false);
SetupCacheEntry(cache.get(), NEG_CACHE_EXPIRED_URL, true, false);
@@ -174,32 +209,63 @@ TEST(CachingV4, InvalidateExpiredCacheEntry)
SetupCacheEntry(cache.get(), BOTH_CACHE_EXPIRED_URL, true, true);
// Before invalidate
TestCache(CACHED_URL, true, true, true, cache.get());
TestCache(NEG_CACHE_EXPIRED_URL, true, true, true, cache.get());
TestCache(POS_CACHE_EXPIRED_URL, true, false, true, cache.get());
TestCache(BOTH_CACHE_EXPIRED_URL, true, false, true, cache.get());
TestCache<T>(CACHED_URL, true, true, true, cache.get());
TestCache<T>(NEG_CACHE_EXPIRED_URL, true, true, true, cache.get());
TestCache<T>(POS_CACHE_EXPIRED_URL, true, false, true, cache.get());
TestCache<T>(BOTH_CACHE_EXPIRED_URL, true, false, true, cache.get());
// Call InvalidateExpiredCacheEntry to remove cache entries whose negative cache
// time is expired
cache->InvalidateExpiredCacheEntry();
cache->InvalidateExpiredCacheEntries();
// After invalidate, NEG_CACHE_EXPIRED_URL & BOTH_CACHE_EXPIRED_URL should
// not be found in cache.
TestCache(NEG_CACHE_EXPIRED_URL, true, false, false, cache.get());
TestCache(BOTH_CACHE_EXPIRED_URL, true, false, false, cache.get());
TestCache<T>(NEG_CACHE_EXPIRED_URL, true, false, false, cache.get());
TestCache<T>(BOTH_CACHE_EXPIRED_URL, true, false, false, cache.get());
// Other entries should remain the same result.
TestCache(CACHED_URL, true, true, true, cache.get());
TestCache(POS_CACHE_EXPIRED_URL, true, false, true, cache.get());
TestCache<T>(CACHED_URL, true, true, true, cache.get());
TestCache<T>(POS_CACHE_EXPIRED_URL, true, false, true, cache.get());
}
TEST(UrlClassifierCaching, InvalidateExpiredCacheEntryV2)
{
TestInvalidateExpiredCacheEntry<LookupCacheV2>();
}
TEST(UrlClassifierCaching, InvalidateExpiredCacheEntryV4)
{
TestInvalidateExpiredCacheEntry<LookupCacheV4>();
}
// This testcase check if an cache entry whose negative cache time is expired
// and it doesn't have any postive cache entries in it, it should be removed
// from cache after calling |Has|.
TEST(CachingV4, NegativeCacheExpire)
TEST(UrlClassifierCaching, NegativeCacheExpireV2)
{
_PrefixArray array = { GeneratePrefix(NEG_CACHE_EXPIRED_URL, 8) };
UniquePtr<LookupCacheV4> cache = SetupLookupCacheV4(array);
UniquePtr<LookupCacheV2> cache = SetupLookupCache<LookupCacheV2>(array);
nsCOMPtr<nsICryptoHash> cryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID);
MissPrefixArray misses;
Prefix* prefix = misses.AppendElement(fallible);
prefix->FromPlaintext(NEG_CACHE_EXPIRED_URL, cryptoHash);
AddCompleteArray dummy;
cache->AddGethashResultToCache(dummy, misses, EXPIRED_TIME_SEC);
// Ensure it is in cache in the first place.
EXPECT_EQ(cache->IsInCache(prefix->ToUint32()), true);
// It should be removed after calling Has API.
TestCache<LookupCacheV2>(NEG_CACHE_EXPIRED_URL, true, false, false, cache.get());
}
TEST(UrlClassifierCaching, NegativeCacheExpireV4)
{
_PrefixArray array = { GeneratePrefix(NEG_CACHE_EXPIRED_URL, 8) };
UniquePtr<LookupCacheV4> cache = SetupLookupCache<LookupCacheV4>(array);
FullHashResponseMap map;
Prefix prefix;
@@ -211,10 +277,9 @@ TEST(CachingV4, NegativeCacheExpire)
cache->AddFullHashResponseToCache(map);
// The first time we should found it in the cache but the result is not
// confirmed(because it is expired).
TestCache(NEG_CACHE_EXPIRED_URL, true, false, true, cache.get());
// Ensure it is in cache in the first place.
EXPECT_EQ(cache->IsInCache(prefix.ToUint32()), true);
// The second time it should not be found in the cache again
TestCache(NEG_CACHE_EXPIRED_URL, true, false, false, cache.get());
// It should be removed after calling Has API.
TestCache<LookupCacheV4>(NEG_CACHE_EXPIRED_URL, true, false, false, cache.get());
}

9
toolkit/components/url-classifier/tests/gtest/TestLookupCacheV4.cpp
View File

@@ -14,24 +14,23 @@ TestHasPrefix(const _Fragment& aFragment, bool aExpectedHas, bool aExpectedCompl
};
RunTestInNewThread([&] () -> void {
UniquePtr<LookupCache> cache = SetupLookupCacheV4(array);
UniquePtr<LookupCache> cache = SetupLookupCache<LookupCacheV4>(array);
Completion lookupHash;
nsCOMPtr<nsICryptoHash> cryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID);
lookupHash.FromPlaintext(aFragment, cryptoHash);
bool has, confirmed, fromCache;
bool has, confirmed;
uint32_t matchLength;
// Freshness is not used in V4 so we just put dummy values here.
TableFreshnessMap dummy;
nsresult rv = cache->Has(lookupHash, dummy, 0,
&has, &matchLength, &confirmed, &fromCache);
nsresult rv =
cache->Has(lookupHash, &has, &matchLength, &confirmed);
EXPECT_EQ(rv, NS_OK);
EXPECT_EQ(has, aExpectedHas);
EXPECT_EQ(matchLength == COMPLETE_SIZE, aExpectedComplete);
EXPECT_EQ(confirmed, false);
EXPECT_EQ(fromCache, false);
cache->ClearAll();
});

2
toolkit/components/url-classifier/tests/gtest/moz.build
View File

@@ -10,7 +10,7 @@ LOCAL_INCLUDES += [
UNIFIED_SOURCES += [
'Common.cpp',
'TestCachingV4.cpp',
'TestCaching.cpp',
'TestChunkSet.cpp',
'TestClassifier.cpp',
'TestFailUpdate.cpp',

134
toolkit/components/url-classifier/tests/mochitest/cache.sjs Normal file
View File

@@ -0,0 +1,134 @@
/* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/
*/
const CC = Components.Constructor;
const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
"nsIBinaryInputStream",
"setInputStream");
function handleRequest(request, response)
{
var query = {};
request.queryString.split('&').forEach(function (val) {
var idx = val.indexOf('=');
query[val.slice(0, idx)] = unescape(val.slice(idx + 1));
});
var responseBody;
// Store fullhash in the server side.
if ("list" in query && "fullhash" in query) {
// In the server side we will store:
// 1. All the full hashes for a given list
// 2. All the lists we have right now
// data is separate by '\n'
let list = query["list"];
let hashes = getState(list);
let hash = base64ToString(query["fullhash"]);
hashes += hash + "\n";
setState(list, hashes);
let lists = getState("lists");
if (lists.indexOf(list) == -1) {
lists += list + "\n";
setState("lists", lists);
}
return;
// gethash count return how many gethash request received.
// This is used by client to know if a gethash request is triggered by gecko
} else if ("gethashcount" == request.queryString) {
var counter = getState("counter");
responseBody = counter == "" ? "0" : counter;
} else {
var body = new BinaryInputStream(request.bodyInputStream);
var avail;
var bytes = [];
while ((avail = body.available()) > 0) {
Array.prototype.push.apply(bytes, body.readByteArray(avail));
}
var counter = getState("counter");
counter = counter == "" ? "1" : (parseInt(counter) + 1).toString();
setState("counter", counter);
responseBody = parseV2Request(bytes);
}
response.setHeader("Content-Type", "text/plain", false);
response.write(responseBody);
}
function parseV2Request(bytes) {
var request = String.fromCharCode.apply(this, bytes);
var [HEADER, PREFIXES] = request.split("\n");
var [PREFIXSIZE, LENGTH] = HEADER.split(":").map(val => {
return parseInt(val);
});
var ret = "";
for(var start = 0; start < LENGTH; start += PREFIXSIZE) {
getState("lists").split("\n").forEach(function(list) {
var completions = getState(list).split("\n");
for (var completion of completions) {
if (completion.indexOf(PREFIXES.substr(start, PREFIXSIZE)) == 0) {
ret += list + ":" + "1" + ":" + "32" + "\n";
ret += completion;
}
}
});
}
return ret;
}
/* Convert Base64 data to a string */
const toBinaryTable = [
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63,
52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1, 0,-1,-1,
-1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14,
15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1,
-1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40,
41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1
];
const base64Pad = '=';
function base64ToString(data) {
var result = '';
var leftbits = 0; // number of bits decoded, but yet to be appended
var leftdata = 0; // bits decoded, but yet to be appended
// Convert one by one.
for (var i = 0; i < data.length; i++) {
var c = toBinaryTable[data.charCodeAt(i) & 0x7f];
var padding = (data[i] == base64Pad);
// Skip illegal characters and whitespace
if (c == -1) continue;
// Collect data into leftdata, update bitcount
leftdata = (leftdata << 6) | c;
leftbits += 6;
// If we have 8 or more bits, append 8 bits to the result
if (leftbits >= 8) {
leftbits -= 8;
// Append if not padding.
if (!padding)
result += String.fromCharCode((leftdata >> leftbits) & 0xff);
leftdata &= (1 << leftbits) - 1;
}
}
// If there are any bits left, the base64 string was corrupted
if (leftbits)
throw Components.Exception('Corrupted base64 string');
return result;
}

2
toolkit/components/url-classifier/tests/mochitest/mochitest.ini
View File

@@ -31,6 +31,7 @@ support-files =
gethashFrame.html
tracker.js
seek.webm
cache.sjs
[test_classifier.html]
skip-if = (os == 'linux' && debug) #Bug 1199778
@@ -40,3 +41,4 @@ skip-if = (os == 'linux' && debug) #Bug 1199778
[test_classify_track.html]
[test_gethash.html]
[test_bug1254766.html]
[test_cachemiss.html]

19
toolkit/components/url-classifier/tests/mochitest/test_bug1254766.html
View File

@@ -146,24 +146,6 @@ function testGethash() {
.then(reset);
}
// This testcase is to make sure an update request will clear completion cache:
// 1. Add prefixes to DB.
// 2. Load test frame, this should trigger a gethash request
// 3. Trigger an update, completion cache should be cleared now.
// 4. Load test frame again, since cache is cleared now, gethash request should be triggered.
function testUpdateClearCache() {
return Promise.resolve()
.then(addPrefixToDB)
.then(loadTestFrame)
.then(() => {
ok(gCurGethashCounter > gPreGethashCounter, "Gethash request is triggered."); })
.then(updateUnusedUrl)
.then(loadTestFrame)
.then(() => {
ok(gCurGethashCounter > gPreGethashCounter, "Gethash request is triggered."); })
.then(reset);
}
// This testcae is to make sure completions in update works:
// 1. Add completions to DB.
// 2. Load test frame, since completions is stored in DB, gethash request should
@@ -239,7 +221,6 @@ function runTest() {
.then(classifierHelper.waitForInit)
.then(setup)
.then(testGethash)
.then(testUpdateClearCache)
.then(testUpdate)
.then(testUpdateNotClearCompletions)
.then(testUpdateCompletionsAfterReload)

161
toolkit/components/url-classifier/tests/mochitest/test_cachemiss.html Normal file
View File

@@ -0,0 +1,161 @@
<!DOCTYPE HTML>
<html>
<head>
<title>Bug 1272239 - Test gethash.</title>
<script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
<script type="text/javascript" src="classifierHelper.js"></script>
<link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
</head>
<body>
<p id="display"></p>
<div id="content" style="display: none">
</div>
<pre id="test">
<script src="head.js"></script>
<script class="testbody" type="text/javascript">
const MALWARE_LIST = "test-malware-simple";
const MALWARE_HOST = "malware.example.com/";
const UNWANTED_LIST = "test-unwanted-simple";
const UNWANTED_HOST = "unwanted.example.com/";
const GETHASH_URL = "http://mochi.test:8888/tests/toolkit/components/url-classifier/tests/mochitest/cache.sjs";
var shouldLoad = false;
var gPreGethashCounter = 0;
var gCurGethashCounter = 0;
function loadTestFrame() {
return new Promise(function(resolve, reject) {
var iframe = document.createElement("iframe");
iframe.setAttribute("src", "gethashFrame.html");
document.body.appendChild(iframe);
iframe.onload = function() {
document.body.removeChild(iframe);
resolve();
};
}).then(getGethashCounter);
}
function getGethashCounter() {
return new Promise(function(resolve, reject) {
var xhr = new XMLHttpRequest;
xhr.open("PUT", GETHASH_URL + "?gethashcount");
xhr.setRequestHeader("Content-Type", "text/plain");
xhr.onreadystatechange = function() {
if (this.readyState == this.DONE) {
gPreGethashCounter = gCurGethashCounter;
gCurGethashCounter = parseInt(xhr.response);
resolve();
}
};
xhr.send();
});
}
// add 4-bytes prefixes to local database, so when we access the url,
// it will trigger gethash request.
function addPrefixToDB(list, url) {
var testData = [{ db: list, url: url, len: 4 }];
return classifierHelper.addUrlToDB(testData)
.catch(function(err) {
ok(false, "Couldn't update classifier. Error code: " + err);
// Abort test.
SimpleTest.finish();
});
}
// manually reset DB to make sure next test won't be affected by cache.
function reset() {
return classifierHelper.resetDB;
}
// This test has to come before testPositiveCache to ensure gethash server doesn't
// contain completions.
function testNegativeCache() {
shouldLoad = true;
function setup() {
classifierHelper.allowCompletion([MALWARE_LIST, UNWANTED_LIST], GETHASH_URL);
// Only add prefix to database. not server, so gethash will not return
// result.
return Promise.all([
addPrefixToDB(MALWARE_LIST, MALWARE_HOST),
addPrefixToDB(UNWANTED_LIST, UNWANTED_HOST),
]);
}
return Promise.resolve()
.then(setup)
.then(() => loadTestFrame())
.then(() => {
ok(gCurGethashCounter > gPreGethashCounter, "Gethash request is triggered."); })
// Second load should not trigger gethash request because cache.
.then(() => loadTestFrame())
.then(() => {
ok(gCurGethashCounter == gPreGethashCounter, "Gethash request is nottriggered."); })
.then(reset);
}
function testPositiveCache() {
shouldLoad = false;
function setup() {
classifierHelper.allowCompletion([MALWARE_LIST, UNWANTED_LIST], GETHASH_URL);
return Promise.all([
addPrefixToDB(MALWARE_LIST, MALWARE_HOST),
addPrefixToDB(UNWANTED_LIST, UNWANTED_HOST),
addCompletionToServer(MALWARE_LIST, MALWARE_HOST, GETHASH_URL),
addCompletionToServer(UNWANTED_LIST, UNWANTED_HOST, GETHASH_URL)
]);
}
return Promise.resolve()
.then(setup)
.then(() => loadTestFrame())
.then(() => {
ok(gCurGethashCounter > gPreGethashCounter, "Gethash request is triggered."); })
// Second load should not trigger gethash request because cache.
.then(() => loadTestFrame())
.then(() => {
ok(gCurGethashCounter == gPreGethashCounter, "Gethash request is nottriggered."); })
.then(reset);
}
function runTest() {
Promise.resolve()
// This test resources get blocked when gethash returns successfully
.then(classifierHelper.waitForInit)
.then(testNegativeCache)
.then(testPositiveCache)
.then(function() {
SimpleTest.finish();
}).catch(function(e) {
ok(false, "Some test failed with error " + e);
SimpleTest.finish();
});
}
SimpleTest.waitForExplicitFinish();
// 'network.predictor.enabled' is disabled because if other testcase load
// evil.js, evil.css ...etc resources, it may cause we load them from cache
// directly and bypass classifier check
SpecialPowers.pushPrefEnv({"set": [
["browser.safebrowsing.malware.enabled", true],
["network.predictor.enabled", false],
["urlclassifier.gethash.timeout_ms", 30000],
]}, runTest);
</script>
</pre>
</body>
</html>

80
toolkit/components/url-classifier/tests/unit/test_partial.js
View File

@@ -672,84 +672,6 @@ function testErrorList()
}
function testStaleList()
{
var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
var update = buildPhishingUpdate(
[
{ "chunkNum" : 1,
"urls" : addUrls
}],
32);
var completer = installCompleter('test-phish-simple', [[1, addUrls]], []);
var assertions = {
"tableData" : "test-phish-simple;a:1",
"urlsExist" : addUrls,
// These are complete urls, and will only be completed if the
// list is stale.
"completerQueried" : [completer, addUrls]
};
// Consider a match stale after one second.
prefBranch.setIntPref("urlclassifier.max-complete-age", 1);
// Apply the update.
doStreamUpdate(update, function() {
// Now the test-phish-simple and test-malware-simple tables are marked
// as fresh. Wait three seconds to make sure the list is marked stale.
new Timer(3000, function() {
// Now the lists should be marked stale. Check assertions.
checkAssertions(assertions, function() {
prefBranch.setIntPref("urlclassifier.max-complete-age", 2700);
runNextTest();
});
}, updateError);
}, updateError);
}
// Same as testStaleList, but verifies that an empty response still
// unconfirms the entry.
function testStaleListEmpty()
{
var addUrls = [ "foo.com/a", "foo.com/b", "bar.com/c" ];
var update = buildPhishingUpdate(
[
{ "chunkNum" : 1,
"urls" : addUrls
}],
32);
var completer = installCompleter('test-phish-simple', [], []);
var assertions = {
"tableData" : "test-phish-simple;a:1",
// None of these should match, because they won't be completed
"urlsDontExist" : addUrls,
// These are complete urls, and will only be completed if the
// list is stale.
"completerQueried" : [completer, addUrls]
};
// Consider a match stale after one second.
prefBranch.setIntPref("urlclassifier.max-complete-age", 1);
// Apply the update.
doStreamUpdate(update, function() {
// Now the test-phish-simple and test-malware-simple tables are marked
// as fresh. Wait three seconds to make sure the list is marked stale.
new Timer(3000, function() {
// Now the lists should be marked stale. Check assertions.
checkAssertions(assertions, function() {
prefBranch.setIntPref("urlclassifier.max-complete-age", 2700);
runNextTest();
});
}, updateError);
}, updateError);
}
// Verify that different lists (test-phish-simple,
// test-malware-simple) maintain their freshness separately.
function testErrorListIndependent()
@@ -815,8 +737,6 @@ function run_test()
testCachedResultsWithExpire,
testCachedResultsUpdate,
testCachedResultsFailure,
testStaleList,
testStaleListEmpty,
testErrorList,
testErrorListIndependent
]);