Bug 1382001 - Part 2: Use a DAFSA for kSTSPreloadList. r=keeler

This switches the STS preload list over to a more compact representation by
using a DAFSA. `getHSTSPreloadList.js` is updated to output data in the gperf
format expected by `make_dafsa.py`. We then add a generated file that gets
created by pumping `nsSTSPreloadList.inc` through `make_dafsa.py`.

`nsSiteSecurityService` is updated to use the DAFSA which either returns -1
(kNotFound) if an entry is not present or (0, 1) indicating whether or not to
use subdomains.

`nsSTSPreloadList.inc` is an automated conversion to the new gperf-like format.

security/manager/ssl/moz.build

@@ -176,6 +176,13 @@ LOCAL_INCLUDES += [
     '!/dist/public/nss',
 ]
 
+GENERATED_FILES = [
+    'nsSTSPreloadList.h',
+]
+dafsa_data = GENERATED_FILES['nsSTSPreloadList.h']
+dafsa_data.script = '../../../xpcom/ds/make_dafsa.py'
+dafsa_data.inputs = ['nsSTSPreloadList.inc']
+
 if CONFIG['NSS_DISABLE_DBM']:
     DEFINES['NSS_DISABLE_DBM'] = '1'
 

security/manager/ssl/nsSiteSecurityService.cpp

@@ -41,7 +41,7 @@
 // influence its HSTS status via include subdomains, however).
 // This prevents the preload list from overriding the site's current
 // desired HSTS status.
-#include "nsSTSPreloadList.inc"
+#include "nsSTSPreloadList.h"
 
 using namespace mozilla;
 using namespace mozilla::psm;
@@ -495,6 +495,7 @@ nsSiteSecurityService::nsSiteSecurityService()
   : mMaxMaxAge(kSixtyDaysInSeconds)
   , mUsePreloadList(true)
   , mPreloadListTimeOffset(0)
+  , mDafsa(kDafsa)
 {
 }
 
@@ -730,7 +731,7 @@ nsSiteSecurityService::RemoveStateInternal(
                                               mozilla::DataStorage_Persistent);
   RefPtr<SiteHSTSState> dynamicState =
     new SiteHSTSState(aHost, aOriginAttributes, value);
-  if (GetPreloadListEntry(aHost.get()) ||
+  if (GetPreloadStatus(aHost) ||
       dynamicState->mHSTSState != SecurityPropertyUnset) {
     SSSLOG(("SSS: storing knockout entry for %s", aHost.get()));
     RefPtr<SiteHSTSState> siteState = new SiteHSTSState(
@@ -1370,29 +1371,31 @@ nsSiteSecurityService::IsSecureURI(uint32_t aType, nsIURI* aURI,
                       source, aResult);
 }
 
-int STSPreloadCompare(const void *key, const void *entry)
+// Checks if the given host is in the preload list.
+//
+// @param aHost The host to match. Only does exact host matching.
+// @param aIncludeSubdomains Out, optional. Indicates whether or not to include
+//        subdomains. Only set if the host is matched and this function returns
+//        true.
+//
+// @return True if the host is matched, false otherwise.
+bool
+nsSiteSecurityService::GetPreloadStatus(const nsACString& aHost,
+                                        bool* aIncludeSubdomains) const
 {
-  const char *keyStr = (const char *)key;
+  const int kIncludeSubdomains = 1;
-  const nsSTSPreload *preloadEntry = (const nsSTSPreload *)entry;
+  bool found = false;
-  return strcmp(keyStr, &kSTSHostTable[preloadEntry->mHostIndex]);
-}
 
-// Returns the preload list entry for the given host, if it exists.
-// Only does exact host matching - the user must decide how to use the returned
-// data. May return null.
-const nsSTSPreload *
-nsSiteSecurityService::GetPreloadListEntry(const char *aHost)
-{
   PRTime currentTime = PR_Now() + (mPreloadListTimeOffset * PR_USEC_PER_SEC);
   if (mUsePreloadList && currentTime < gPreloadListExpirationTime) {
-    return (const nsSTSPreload *) bsearch(aHost,
+    int result = mDafsa.Lookup(aHost);
-                                          kSTSPreloadList,
+    found = (result != mozilla::Dafsa::kKeyNotFound);
-                                          mozilla::ArrayLength(kSTSPreloadList),
+    if (found && aIncludeSubdomains) {
-                                          sizeof(nsSTSPreload),
+      *aIncludeSubdomains = (result == kIncludeSubdomains);
-                                          STSPreloadCompare);
+    }
   }
 
-  return nullptr;
+  return found;
 }
 
 // Allows us to determine if we have an HSTS entry for a given host (and, if
@@ -1476,7 +1479,7 @@ nsSiteSecurityService::HostHasHSTSEntry(
       if (dynamicState->mHSTSState == SecurityPropertyUnset) {
         SSSLOG(("No dynamic preload - checking for static preload"));
         // Now check the static preload list.
-        if (!GetPreloadListEntry(aHost.get())) {
+        if (!GetPreloadStatus(aHost)) {
           SSSLOG(("No static preload - removing expired entry"));
           mSiteStateStorage->Remove(storageKey, storageType);
         }
@@ -1520,7 +1523,7 @@ nsSiteSecurityService::HostHasHSTSEntry(
     } else {
       // if a dynamic preload has expired and is not in the static preload
       // list, we can remove it.
-      if (!GetPreloadListEntry(aHost.get())) {
+      if (!GetPreloadStatus(aHost)) {
         mPreloadStateStorage->Remove(preloadKey,
                                      mozilla::DataStorage_Persistent);
       }
@@ -1528,18 +1531,18 @@ nsSiteSecurityService::HostHasHSTSEntry(
     return false;
   }
 
-  const nsSTSPreload* preload = nullptr;
+  bool includeSubdomains = false;
 
   // Finally look in the static preload list.
   if (siteState->mHSTSState == SecurityPropertyUnset &&
       dynamicState->mHSTSState == SecurityPropertyUnset &&
-      (preload = GetPreloadListEntry(aHost.get())) != nullptr) {
+      GetPreloadStatus(aHost, &includeSubdomains)) {
     SSSLOG(("%s is a preloaded HSTS host", aHost.get()));
-    *aResult = aRequireIncludeSubdomains ? preload->mIncludeSubdomains
+    *aResult = aRequireIncludeSubdomains ? includeSubdomains
                                          : true;
     if (aCached) {
       // Only set cached if this includes subdomains
-      *aCached = aRequireIncludeSubdomains ? preload->mIncludeSubdomains
+      *aCached = aRequireIncludeSubdomains ? includeSubdomains
                                            : true;
     }
     if (aSource) {

security/manager/ssl/nsSiteSecurityService.h

@@ -6,6 +6,7 @@
 #define __nsSiteSecurityService_h__
 
 #include "mozilla/BasePrincipal.h"
+#include "mozilla/Dafsa.h"
 #include "mozilla/DataStorage.h"
 #include "mozilla/RefPtr.h"
 #include "nsCOMPtr.h"
@@ -208,7 +209,8 @@ private:
                         const OriginAttributes& aOriginAttributes,
                         bool* aResult, bool* aCached,
                         SecurityPropertySource* aSource);
-  const nsSTSPreload *GetPreloadListEntry(const char *aHost);
+  bool GetPreloadStatus(const nsACString& aHost,
+                        /*optional out*/ bool* aIncludeSubdomains = nullptr) const;
   nsresult IsSecureHost(uint32_t aType, const nsACString& aHost,
                         uint32_t aFlags,
                         const OriginAttributes& aOriginAttributes,
@@ -221,6 +223,7 @@ private:
   bool mProcessPKPHeadersFromNonBuiltInRoots;
   RefPtr<mozilla::DataStorage> mSiteStateStorage;
   RefPtr<mozilla::DataStorage> mPreloadStateStorage;
+  const mozilla::Dafsa mDafsa;
 };
 
 #endif // __nsSiteSecurityService_h__

security/manager/tools/getHSTSPreloadList.js

@@ -42,6 +42,7 @@ const HEADER = "/* This Source Code Form is subject to the terms of the Mozilla
 "/*****************************************************************************/\n" +
 "\n" +
 "#include <stdint.h>\n";
+const GPERF_DELIM = "%%\n";
 
 function download() {
   var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
@@ -240,12 +241,6 @@ function errorToString(status) {
           : status.error);
 }
 
-function writeEntry(status, indices, outputStream) {
-  let includeSubdomains = (status.finalIncludeSubdomains ? "true" : "false");
-  writeTo("  { " + indices[status.name] + ", " + includeSubdomains + " },\n",
-          outputStream);
-}
-
 function output(sortedStatuses, currentList) {
   try {
     var file = FileUtils.getFile("CurWorkD", [OUTPUT]);
@@ -298,47 +293,14 @@ function output(sortedStatuses, currentList) {
       status.finalIncludeSubdomains = incSubdomainsBool;
     }
 
-    writeTo("\nstatic const char kSTSHostTable[] = {\n", fos);
+    writeTo(GPERF_DELIM, fos);
-    var indices = {};
-    var currentIndex = 0;
-    for (let status of includedStatuses) {
-      indices[status.name] = currentIndex;
-      // Add 1 for the null terminator in C.
-      currentIndex += status.name.length + 1;
-      // Rebuilding the preload list requires reading the previous preload
-      // list.  Write out a comment describing each host prior to writing out
-      // the string for the host.
-      writeTo("  /* \"" + status.name + "\", " +
-              (status.finalIncludeSubdomains ? "true" : "false") + " */ ",
-              fos);
-      // Write out the string itself as individual characters, including the
-      // null terminator.  We do it this way rather than using C's string
-      // concatentation because some compilers have hardcoded limits on the
-      // lengths of string literals, and the preload list is large enough
-      // that it runs into said limits.
-      for (let c of status.name) {
-        writeTo("'" + c + "', ", fos);
-      }
-      writeTo("'\\0',\n", fos);
-    }
-    writeTo("};\n", fos);
 
-    const PREFIX = "\n" +
-      "struct nsSTSPreload\n" +
-      "{\n" +
-      "  // See bug 1338873 about making these fields const.\n" +
-      "  uint32_t mHostIndex : 31;\n" +
-      "  uint32_t mIncludeSubdomains : 1;\n" +
-      "};\n" +
-      "\n" +
-      "static const nsSTSPreload kSTSPreloadList[] = {\n";
-    const POSTFIX = "};\n";
-
-    writeTo(PREFIX, fos);
     for (let status of includedStatuses) {
-      writeEntry(status, indices, fos);
+      let includeSubdomains = (status.finalIncludeSubdomains ? 1 : 0);
+      writeTo(status.name + ", " + includeSubdomains + "\n", fos);
     }
-    writeTo(POSTFIX, fos);
+
+    writeTo(GPERF_DELIM, fos);
     FileUtils.closeSafeFileOutputStream(fos);
     FileUtils.closeSafeFileOutputStream(eos);
   } catch (e) {
@@ -398,19 +360,22 @@ function readCurrentList(filename) {
               .createInstance(Ci.nsILineInputStream);
   fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF);
   var line = {};
-  // While we generate entries matching the version 2 format (see bug 1255425
+
-  // for details), we still need to be able to read entries in the version 1
+  // While we generate entries matching the latest version format,
-  // format for bootstrapping a version 2 preload list from a version 1
+  // we still need to be able to read entries in the previous version formats
-  // preload list.  Hence these two regexes.
+  // for bootstrapping a latest version preload list from a previous version
-  var v1EntryRegex = / {2}{ "([^"]*)", (true|false) },/;
+  // preload list. Hence these regexes.
-  var v2EntryRegex = / {2}\/\* "([^"]*)", (true|false) \*\//;
+  const entryRegexes = [
+    /([^,]+), (0|1)/,                         // v3
+    / {2}\/\* "([^"]*)", (true|false) \*\//,  // v2
+    / {2}{ "([^"]*)", (true|false) },/,       // v1
+  ];
+
   while (fis.readLine(line)) {
-    var match = v1EntryRegex.exec(line.value);
+    let match;
-    if (!match) {
+    entryRegexes.find((r) => { match = r.exec(line.value); return match; });
-      match = v2EntryRegex.exec(line.value);
-    }
     if (match) {
-      currentHosts[match[1]] = (match[2] == "true");
+      currentHosts[match[1]] = (match[2] == "1" || match[2] == "true");
     }
   }
   return currentHosts;