Bug 1889988 - Added remote settings input to defaultUri bypass list r=necko-reviewers,valentin,nika,Standard8

Differential Revision: https://phabricator.services.mozilla.com/D207810

dom/ipc/ContentChild.cpp

@@ -2398,6 +2398,13 @@ mozilla::ipc::IPCResult ContentChild::RecvRegisterStringBundles(
   return IPC_OK();
 }
 
+mozilla::ipc::IPCResult ContentChild::RecvSimpleURIUnknownRemoteSchemes(
+    nsTArray<nsCString>&& aRemoteSchemes) {
+  RefPtr<nsIOService> io = nsIOService::GetInstance();
+  io->SetSimpleURIUnknownRemoteSchemes(aRemoteSchemes);
+  return IPC_OK();
+}
+
 mozilla::ipc::IPCResult ContentChild::RecvUpdateL10nFileSources(
     nsTArray<mozilla::dom::L10nFileSourceDescriptor>&& aDescriptors) {
   L10nRegistry::RegisterFileSourcesFromParentProcess(aDescriptors);

dom/ipc/ContentChild.h

@@ -310,6 +310,9 @@ class ContentChild final : public PContentChild,
   mozilla::ipc::IPCResult RecvRegisterStringBundles(
       nsTArray<StringBundleDescriptor>&& stringBundles);
 
+  mozilla::ipc::IPCResult RecvSimpleURIUnknownRemoteSchemes(
+      nsTArray<nsCString>&& aRemoteSchemes);
+
   mozilla::ipc::IPCResult RecvUpdateL10nFileSources(
       nsTArray<L10nFileSourceDescriptor>&& aDescriptors);
 

dom/ipc/ContentParent.cpp

@@ -1635,6 +1635,13 @@ void ContentParent::Init() {
   mQueuedPrefs.Clear();
 
   Unused << SendInitNextGenLocalStorageEnabled(NextGenLocalStorageEnabled());
+
+  // sending only the remote settings schemes to the content process
+  nsCOMPtr<nsIIOService> io(do_GetIOService());
+  MOZ_ASSERT(io, "No IO service for SimpleURI scheme broadcast to content");
+  nsTArray<nsCString> remoteSchemes;
+  MOZ_ALWAYS_SUCCEEDS(io->GetSimpleURIUnknownRemoteSchemes(remoteSchemes));
+  Unused << SendSimpleURIUnknownRemoteSchemes(std::move(remoteSchemes));
 }
 
 void ContentParent::AsyncSendShutDownMessage() {

dom/ipc/PContent.ipdl

@@ -750,6 +750,8 @@ child:
 
     async RegisterStringBundles(StringBundleDescriptor[] stringBundles);
 
+    async SimpleURIUnknownRemoteSchemes(nsCString[] remoteSchemes);
+
     async UpdateSharedData(SharedMemoryHandle aMapHandle, uint32_t aSize,
                            IPCBlob[] blobs,
                            nsCString[] changedKeys);

modules/libpref/init/StaticPrefList.yaml

@@ -13750,14 +13750,14 @@
   mirror: always
 
 # Allows use of a protocol exception list that will bypass defaultURI parser
-- name: network.url.some_schemes_bypass_defaultURI_fallback
+- name: network.url.simple_uri_unknown_schemes_enabled
   type: RelaxedAtomicBool
   value: true
   mirror: always
 
 # A list of schemes to allow for bypassing defaultURI as default
-# This is only used when network.url.some_schemes_bypass_defaultURI_fallback is true
+# This is only used when network.url.simple_uri_unknown_schemes_enabled is true
-- name: network.url.simple_uri_schemes
+- name: network.url.simple_uri_unknown_schemes
   type: String
   value: ""
   mirror: never

netwerk/base/SimpleURIUnknownSchemes.cpp

@@ -0,0 +1,74 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=4 sw=2 sts=2 et cin: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "SimpleURIUnknownSchemes.h"
+#include "mozilla/StaticPrefs_network.h"
+
+static mozilla::LazyLogModule gURLLog("URL");
+
+namespace mozilla::net {
+
+nsTArray<nsCString> ParseUriSchemes(const nsCString& inputStrList) {
+  nsTArray<nsCString> result;
+  for (const auto& scheme : inputStrList.Split(',')) {
+    nsCString* str = result.AppendElement(scheme);
+    str->StripWhitespace();
+  }
+  return result;
+}
+
+void SimpleURIUnknownSchemes::ParseAndMergePrefSchemes() {
+  AutoWriteLock lock(mSchemeLock);
+  ParseAndMergePrefSchemesLocked();
+}
+
+void SimpleURIUnknownSchemes::ParseAndMergePrefSchemesLocked() {
+  nsAutoCString prefListStr;
+  Preferences::GetCString(SIMPLE_URI_SCHEMES_PREF, prefListStr);
+  nsTArray<nsCString> prefSchemes = ParseUriSchemes(prefListStr);
+  MergeSimpleURISchemes(prefSchemes, mRemoteSettingsURISchemes);
+}
+
+void SimpleURIUnknownSchemes::SetAndMergeRemoteSchemes(
+    const nsTArray<nsCString>& remoteSettingsList) {
+  MOZ_LOG(gURLLog, LogLevel::Debug,
+          ("SimpleURIUnknownSchemes::SetAndMergeRemoteSchemes()"));
+  AutoWriteLock lock(mSchemeLock);
+
+  // update the local copy of remote settings schemes in case of pref-update
+  mRemoteSettingsURISchemes = remoteSettingsList.Clone();
+
+  // update the merged list with the new remote settings schemes
+  ParseAndMergePrefSchemesLocked();
+}
+
+void SimpleURIUnknownSchemes::MergeSimpleURISchemes(
+    const nsTArray<nsCString>& prefList,
+    const nsTArray<nsCString>& remoteSettingsList) {
+  mSimpleURISchemes.Clear();
+  for (const nsCString& scheme : prefList) {
+    mSimpleURISchemes.Insert(scheme);
+  }
+  for (const nsCString& scheme : remoteSettingsList) {
+    mSimpleURISchemes.Insert(scheme);
+  }
+}
+
+bool SimpleURIUnknownSchemes::IsSimpleURIUnknownScheme(
+    const nsACString& aScheme) {
+  AutoReadLock lock(mSchemeLock);
+  return mSimpleURISchemes.Contains(aScheme);
+}
+
+void SimpleURIUnknownSchemes::GetRemoteSchemes(nsTArray<nsCString>& aArray) {
+  aArray.Clear();
+  AutoReadLock lock(mSchemeLock);
+  for (const auto& uri : mRemoteSettingsURISchemes) {
+    aArray.AppendElement(uri);
+  }
+}
+
+}  // namespace mozilla::net

netwerk/base/SimpleURIUnknownSchemes.h

@@ -0,0 +1,53 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim:set ts=4 sw=2 sts=2 et cin: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef SimpleURIUnknownSchemes_h__
+#define SimpleURIUnknownSchemes_h__
+
+#include "nsString.h"
+#include "mozilla/RWLock.h"
+#include "nsTArray.h"
+#include "nsTHashSet.h"
+
+#define SIMPLE_URI_SCHEMES_PREF "network.url.simple_uri_unknown_schemes"
+
+namespace mozilla::net {
+
+class SimpleURIUnknownSchemes {
+ public:
+  SimpleURIUnknownSchemes() = default;
+
+  // parse the list in the pref specified by SIMPLE_URI_SCHEMES_PREF
+  // then merge them with the list obtained from remote settings
+  // into a list to tell URL constructors of unknown schemes
+  // to use our simpleURI parser
+  void ParseAndMergePrefSchemes();
+
+  // pass a remote-settings specified list of unknown schemes that should use
+  // the simple uri parser
+  // store a local copy of the list
+  // and merge the list with the pref-specified list of unknown schemes
+  void SetAndMergeRemoteSchemes(const nsTArray<nsCString>& remoteSettingsList);
+
+  bool IsSimpleURIUnknownScheme(const nsACString& aScheme);
+  void GetRemoteSchemes(nsTArray<nsCString>& aArray);
+
+ private:
+  void ParseAndMergePrefSchemesLocked() MOZ_REQUIRES(mSchemeLock);
+  void MergeSimpleURISchemes(const nsTArray<nsCString>& prefList,
+                             const nsTArray<nsCString>& remoteSettingsList)
+      MOZ_REQUIRES(mSchemeLock);
+
+  mutable RWLock mSchemeLock{"SimpleURIUnknownSchemes"};
+  nsTHashSet<nsCString> mSimpleURISchemes MOZ_GUARDED_BY(mSchemeLock);
+
+  // process-local copy of the remote settings schemes
+  // keep them separate from pref-entered schemes so user cannot overwrite
+  nsTArray<nsCString> mRemoteSettingsURISchemes MOZ_GUARDED_BY(mSchemeLock);
+};
+
+}  // namespace mozilla::net
+#endif  // SimpleURIUnknownSchemes_h__

netwerk/base/SimpleURIUnknownSchemesRemoteObserver.sys.mjs

@@ -0,0 +1,118 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const lazy = {};
+
+ChromeUtils.defineESModuleGetters(lazy, {
+  RemoteSettings: "resource://services-settings/remote-settings.sys.mjs",
+  RemoteSettingsClient:
+    "resource://services-settings/RemoteSettingsClient.sys.mjs",
+});
+
+const SETTINGS_DEFAULTURI_BYPASS_LIST_KEY =
+  "url-parser-default-unknown-schemes-interventions";
+
+export class SimpleURIUnknownSchemesRemoteObserver {
+  #initialized = false;
+  #bypassListSettings;
+  classID = Components.ID("{86606ba1-de17-4df4-9013-e571ab94fd94}");
+  QueryInterface = ChromeUtils.generateQI([
+    "nsIObserver",
+    "nsISimpleURIUnknownSchemesRemoteObserver",
+  ]);
+
+  observe(subject, topic) {
+    // signal selected because RemoteSettingsClient is first getting initialised
+    // by the AddonManager at addons-startup
+    if (topic == "profile-after-change" && !this.#initialized) {
+      this.#initialized = true;
+      this.#init();
+    }
+  }
+
+  /**
+   * This method updates the io service with the local scheme list used to
+   * bypass the defaultURI parser and use the simpleURI parser.
+   * It also subscribes to Remote Settings changes to this list which are then
+   * broadcast to processes interested in URL parsing.
+   *
+   * note that there doesn't appear to be a way to get a URI with a non-special
+   * scheme into about:preferences so it should be safe to spin this up early
+   */
+  async #init() {
+    if (!this.#bypassListSettings) {
+      this.#bypassListSettings = lazy.RemoteSettings(
+        SETTINGS_DEFAULTURI_BYPASS_LIST_KEY
+      );
+    }
+
+    // Trigger a get from local remote settings and update the io service.
+    const settingsList = await this.#getBypassList();
+    let schemes = settingsList.map(r => r.scheme);
+    if (schemes.length) {
+      Services.io.setSimpleURIUnknownRemoteSchemes(schemes);
+    }
+
+    // Listen for future updates after we first get the values.
+    this.#bypassListSettings.on("sync", this.#updateBypassList.bind(this));
+  }
+
+  async #updateBypassList() {
+    const settingsList = await this.#getBypassList();
+    let schemes = settingsList.map(r => r.scheme);
+    if (schemes.length) {
+      Services.io.setSimpleURIUnknownRemoteSchemes(schemes);
+    }
+  }
+
+  async #getBypassList() {
+    if (this._getSettingsPromise) {
+      return this._getSettingsPromise;
+    }
+
+    const settings = await (this._getSettingsPromise =
+      this.#getBypassListSettings());
+    delete this._getSettingsPromise;
+    return settings;
+  }
+
+  /**
+   * Obtains the current bypass list from remote settings. This includes
+   * verifying the signature of the bypass list within the database.
+   *
+   * If the signature in the database is invalid, the database will be wiped
+   * and the stored dump will be used, until the settings next update.
+   *
+   * Note that this may cause a network check of the certificate, but that
+   * should generally be quick.
+   *
+   * @param {boolean} [firstTime]
+   *   Internal boolean to indicate if this is the first time check or not.
+   * @returns {array}
+   *   An array of objects in the database, or an empty array if none
+   *   could be obtained.
+   */
+  async #getBypassListSettings(firstTime = true) {
+    let result = [];
+    try {
+      result = await this.#bypassListSettings.get({
+        verifySignature: true,
+      });
+    } catch (ex) {
+      if (
+        ex instanceof lazy.RemoteSettingsClient.InvalidSignatureError &&
+        firstTime
+      ) {
+        // The local database is invalid, try and reset it.
+        await this.#bypassListSettings.db.clear();
+        // Now call this again.
+        return this.#getBypassListSettings(false);
+      }
+      // Don't throw an error just log it, just continue with no data, and hopefully
+      // a sync will fix things later on.
+      console.error(ex);
+    }
+    return result;
+  }
+}

netwerk/base/moz.build

@@ -172,6 +172,7 @@ EXPORTS.mozilla.net += [
     "RedirectChannelRegistrar.h",
     "RequestContextService.h",
     "SimpleChannelParent.h",
+    "SimpleURIUnknownSchemes.h",
     "SSLTokensCache.h",
     "ThrottleQueue.h",
 ]
@@ -242,6 +243,7 @@ UNIFIED_SOURCES += [
     "SimpleBuffer.cpp",
     "SimpleChannel.cpp",
     "SimpleChannelParent.cpp",
+    "SimpleURIUnknownSchemes.cpp",
     "SSLTokensCache.cpp",
     "ThrottleQueue.cpp",
     "Tickler.cpp",
@@ -290,6 +292,7 @@ else:
 
 EXTRA_JS_MODULES += [
     "NetUtil.sys.mjs",
+    "SimpleURIUnknownSchemesRemoteObserver.sys.mjs",
 ]
 
 DIRS += ["mozurl", "rust-helper", "http-sfv", "idna_glue"]

netwerk/base/nsIIOService.idl

@@ -331,6 +331,29 @@ interface nsIIOService : nsISupports
      * @param aScheme the scheme to unregister a handler for.
      */
     void unregisterProtocolHandler(in ACString aScheme);
+
+    /**
+     * Updates the RemoteSettings-specified portion of the defaultURI bypass
+     * scheme list. The list is then merged with the user-specified pref list
+     * before broadcasting to all alive content processes that may need for URL
+     * parsing.
+     */
+    void setSimpleURIUnknownRemoteSchemes(in Array<ACString> aRemoteSchemes);
+
+    /**
+     * Checks if the provided scheme is in the list of unknown schemes that
+     * should use simpleURI as it's default parser. Where "unknown" scheme means
+     * non-special and otherwise non-common shemes like:
+     * http, about, jar, blob, ssh, etc
+     * See netwerk/base/nsNetUtil.cpp::NS_NewURI for the full list
+     */
+    [noscript] boolean isSimpleURIUnknownScheme(in ACString aScheme);
+
+    /**
+     * returns an array of the remote-settings specified unknown schemes that
+     * should use SimpleURI parser instead of defaultURI parser.
+     */
+    [noscript] Array<ACString> getSimpleURIUnknownRemoteSchemes();
 };
 
 %{C++

netwerk/base/nsIOService.cpp

@@ -98,7 +98,6 @@ using mozilla::dom::ServiceWorkerDescriptor;
 #define WEBRTC_PREF_PREFIX "media.peerconnection."
 #define NETWORK_DNS_PREF "network.dns."
 #define FORCE_EXTERNAL_PREF_PREFIX "network.protocol-handler.external."
-#define SIMPLE_URI_SCHEMES_PREF "network.url.simple_uri_schemes"
 
 nsIOService* gIOService;
 static bool gHasWarnedUploadChannel2;
@@ -1619,11 +1618,9 @@ void nsIOService::PrefsChanged(const char* pref) {
 
   if (!pref || strncmp(pref, SIMPLE_URI_SCHEMES_PREF,
                        strlen(SIMPLE_URI_SCHEMES_PREF)) == 0) {
-    LOG((
+    LOG(("simple_uri_unknown_schemes pref changed, updating the scheme list"));
-        "simple_uri_schemes pref change observed, updating the scheme list\n"));
+    mSimpleURIUnknownSchemes.ParseAndMergePrefSchemes();
-    nsAutoCString schemeList;
+    // runs on parent and child, no need to broadcast
-    Preferences::GetCString(SIMPLE_URI_SCHEMES_PREF, schemeList);
-    mozilla::net::ParseSimpleURISchemes(schemeList);
   }
 }
 
@@ -2290,5 +2287,40 @@ nsIOService::UnregisterProtocolHandler(const nsACString& aScheme) {
              : NS_ERROR_FACTORY_NOT_REGISTERED;
 }
 
+NS_IMETHODIMP
+nsIOService::SetSimpleURIUnknownRemoteSchemes(
+    const nsTArray<nsCString>& aRemoteSchemes) {
+  LOG(("nsIOService::SetSimpleUriUnknownRemoteSchemes"));
+  mSimpleURIUnknownSchemes.SetAndMergeRemoteSchemes(aRemoteSchemes);
+
+  if (XRE_IsParentProcess()) {
+    // since we only expect socket, parent and content processes to create URLs
+    // that need to check the bypass list
+    // we only broadcast the list to content processes
+    // (and leave socket process broadcast as todo if necessary)
+    //
+    // sending only the remote-settings schemes to the content,
+    // which already has the pref list
+    for (auto* cp : mozilla::dom::ContentParent::AllProcesses(
+             mozilla::dom::ContentParent::eLive)) {
+      Unused << cp->SendSimpleURIUnknownRemoteSchemes(aRemoteSchemes);
+    }
+  }
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsIOService::IsSimpleURIUnknownScheme(const nsACString& aScheme,
+                                      bool* _retval) {
+  *_retval = mSimpleURIUnknownSchemes.IsSimpleURIUnknownScheme(aScheme);
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsIOService::GetSimpleURIUnknownRemoteSchemes(nsTArray<nsCString>& _retval) {
+  mSimpleURIUnknownSchemes.GetRemoteSchemes(_retval);
+  return NS_OK;
+}
+
 }  // namespace net
 }  // namespace mozilla

netwerk/base/nsIOService.h

@@ -27,6 +27,7 @@
 #include "nsTHashSet.h"
 #include "nsWeakReference.h"
 #include "nsNetCID.h"
+#include "SimpleURIUnknownSchemes.h"
 
 // We don't want to expose this observer topic.
 // Intended internal use only for remoting offline/inline events.
@@ -267,6 +268,8 @@ class nsIOService final : public nsIIOService,
 
   nsCOMPtr<nsIObserverService> mObserverService;
 
+  SimpleURIUnknownSchemes mSimpleURIUnknownSchemes;
+
  public:
   // Used for all default buffer sizes that necko allocates.
   static uint32_t gDefaultSegmentSize;

netwerk/base/nsNetUtil.cpp

@@ -1806,29 +1806,13 @@ class TlsAutoIncrement {
   T& mVar;
 };
 
-static nsTHashSet<nsCString> sSimpleURISchemes;
-static StaticRWLock sSchemeLock;
-
-namespace mozilla::net {
-
-void ParseSimpleURISchemes(const nsACString& schemeList) {
-  StaticAutoWriteLock lock(sSchemeLock);
-
-  sSimpleURISchemes.Clear();
-  for (const auto& scheme : schemeList.Split(',')) {
-    nsAutoCString s(scheme);
-    s.CompressWhitespace();
-    if (!s.IsEmpty()) {
-      sSimpleURISchemes.Insert(s);
-    }
-  }
-}
-
-}  // namespace mozilla::net
-
 nsresult NS_NewURI(nsIURI** aURI, const nsACString& aSpec,
                    const char* aCharset /* = nullptr */,
                    nsIURI* aBaseURI /* = nullptr */) {
+  // we don't expect any other processes than: socket, content or parent
+  // to be able to create a URL
+  MOZ_ASSERT(XRE_IsSocketProcess() || XRE_IsContentProcess() ||
+             XRE_IsParentProcess());
   TlsAutoIncrement<decltype(gTlsURLRecursionCount)> inc(gTlsURLRecursionCount);
   if (inc.value() >= MAX_RECURSION_COUNT) {
     return NS_ERROR_MALFORMED_URI;
@@ -2022,11 +2006,14 @@ nsresult NS_NewURI(nsIURI** aURI, const nsACString& aSpec,
 #endif
 
   auto mustUseSimpleURI = [](const nsCString& scheme) -> bool {
-    if (!StaticPrefs::network_url_some_schemes_bypass_defaultURI_fallback()) {
+    if (!StaticPrefs::network_url_simple_uri_unknown_schemes_enabled()) {
       return false;
     }
-    StaticAutoReadLock lock(sSchemeLock);
+
-    return sSimpleURISchemes.Contains(scheme);
+    bool res = false;
+    RefPtr<nsIIOService> ios = do_GetIOService();
+    MOZ_ALWAYS_SUCCEEDS(ios->IsSimpleURIUnknownScheme(scheme, &res));
+    return res;
   };
 
   if (aBaseURI) {

netwerk/base/nsNetUtil.h

@@ -1188,8 +1188,6 @@ void CheckForBrokenChromeURL(nsILoadInfo* aLoadInfo, nsIURI* aURI);
 
 bool IsCoepCredentiallessEnabled(bool aIsOriginTrialCoepCredentiallessEnabled);
 
-void ParseSimpleURISchemes(const nsACString& schemeList);
-
 }  // namespace net
 }  // namespace mozilla
 

netwerk/base/nsStandardURL.cpp

@@ -79,8 +79,7 @@ static NS_DEFINE_CID(kThisImplCID, NS_THIS_STANDARDURL_IMPL_CID);
 // can be safely used on other threads.
 StaticRefPtr<nsIIDNService> nsStandardURL::gIDN;
 
-// This value will only be updated on the main thread once.
+Atomic<bool, Relaxed> nsStandardURL::gInitialized{false};
-static Atomic<bool, Relaxed> gInitialized{false};
 
 const char nsStandardURL::gHostLimitDigits[] = {'/', '\\', '?', '#', 0};
 

netwerk/base/nsStandardURL.h

@@ -337,6 +337,9 @@ class nsStandardURL : public nsIFileURL,
   // Checks if the URL has a valid representation.
   bool IsValid();
 
+  // This value will only be updated on the main thread once.
+  static Atomic<bool, Relaxed> gInitialized;
+
   // mSpec contains the normalized version of the URL spec (UTF-8 encoded).
   nsCString mSpec;
   int32_t mDefaultPort{-1};

netwerk/build/components.conf

@@ -759,6 +759,15 @@ Classes = [
         'constructor': 'mozilla::net::CookieJarSettings::CreateForXPCOM',
         'headers': ['mozilla/net/CookieJarSettings.h'],
     },
+    {
+        'cid': '{86606ba1-de17-4df4-9013-e571ab94fd94}',
+        'contract_ids': ['@mozilla.org/SimpleURIUnknownSchemesRemoteObserver;1'],
+        'singleton': True,
+        'esModule': 'resource://gre/modules/SimpleURIUnknownSchemesRemoteObserver.sys.mjs',
+        'constructor': 'SimpleURIUnknownSchemesRemoteObserver',
+        'categories': {'profile-after-change': 'SimpleURIUnknownSchemesRemoteObserver'},
+        'processes': ProcessSelector.MAIN_PROCESS_ONLY,
+    },
 ]
 
 if defined('NECKO_WIFI'):

netwerk/test/browser/browser.toml

@@ -67,6 +67,7 @@ support-files = [
   "cookie_filtering_secure_resource_org.html^headers^",
   "cookie_filtering_square.png",
   "cookie_filtering_square.png^headers^",
+  "simple_unknown_uri_helpers.sys.mjs",
   "x_frame_options.html",
   "x_frame_options.html^headers^",
   "test_1629307.html",
@@ -187,6 +188,10 @@ skip-if = ["socketprocess_networking"] # Bug 1772209
 
 ["browser_set_response_override.js"]
 
+["browser_simple_unknown_uris.js"]
+
+["browser_simple_unknown_uris_sync.js"]
+
 ["browser_speculative_connection_link_header.js"]
 
 ["browser_test_data_channel_observer.js"]

netwerk/test/browser/browser_simple_unknown_uris.js

@@ -0,0 +1,103 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+ChromeUtils.defineESModuleGetters(this, {
+  RemoteSettings: "resource://services-settings/remote-settings.sys.mjs",
+});
+
+const {
+  checkInputAndSerializationMatch,
+  checkInputAndSerializationMatchChild,
+  checkSerializationMissingSecondColon,
+  checkSerializationMissingSecondColonChild,
+  removeSecondColon,
+} = ChromeUtils.importESModule(
+  "resource://testing-common/simple_unknown_uri_helpers.sys.mjs"
+);
+
+add_setup(async () => {
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      ["network.url.useDefaultURI", true],
+      ["network.url.simple_uri_unknown_schemes_enabled", true],
+      ["network.url.simple_uri_unknown_schemes", "simpleprotocol,otherproto"],
+    ],
+  });
+});
+
+add_task(async function test_bypass_remote_settings_static_parent() {
+  // sanity check
+  checkInputAndSerializationMatch("https://example.com/");
+
+  // nsStandardURL removes second colon when nesting protocols
+  checkSerializationMissingSecondColon("https://https://example.com/");
+
+  // no-bypass unknown protocol uses defaultURI
+  checkSerializationMissingSecondColon(
+    "nonsimpleprotocol://https://example.com"
+  );
+
+  // simpleURI keeps the second colon
+  // an unknown protocol in the bypass list will use simpleURI
+  // despite network.url.useDefaultURI being enabled
+  let same = "simpleprotocol://https://example.com";
+  checkInputAndSerializationMatch(same);
+
+  // scheme bypass from static remote-settings
+  checkInputAndSerializationMatch("ed2k://https://example.com");
+
+  // check the pref-specified scheme again (remote settings shouldn't overwrite)
+  checkInputAndSerializationMatch(same);
+});
+
+add_task(async function test_bypass_remote_settings_static_child() {
+  const URL_EXAMPLE = "https://example.com";
+  const tab = BrowserTestUtils.addTab(gBrowser, URL_EXAMPLE);
+  const browser = gBrowser.getBrowserForTab(tab);
+  await BrowserTestUtils.browserLoaded(browser);
+
+  await SpecialPowers.spawn(
+    browser,
+
+    [
+      removeSecondColon.toString(),
+      checkSerializationMissingSecondColonChild.toString(),
+      checkInputAndSerializationMatchChild.toString(),
+    ],
+    (rscSource, csmscSource, ciasmcSource) => {
+      /* eslint-disable no-eval */
+      // eslint-disable-next-line no-unused-vars
+      let removeSecondColon = eval(`(${rscSource})`); // used by check fns
+      let checkSerializationMissingSecondColonChild = eval(`(${csmscSource})`);
+      let checkInputAndSerializationMatchChild = eval(`(${ciasmcSource})`);
+      /* eslint-enable no-eval */
+
+      checkInputAndSerializationMatchChild("https://example.com/");
+
+      // nsStandardURL removes second colon when nesting protocols
+      checkSerializationMissingSecondColonChild("https://https://example.com");
+
+      // no-bypass protocol uses defaultURI
+      checkSerializationMissingSecondColonChild(
+        "nonsimpleprotocol://https://example.com"
+      );
+
+      // simpleURI keeps the second colon
+      // an unknown protocol in the bypass list will use simpleURI
+      // despite network.url.useDefaultURI being enabled
+      let same = "simpleprotocol://https://example.com";
+      checkInputAndSerializationMatchChild(same);
+
+      // scheme bypass from static remote-settings
+      checkInputAndSerializationMatchChild("ed2k://https://example.com");
+
+      // pref-specified scheme shouldn't be overwritten by remote settings schemes
+      checkInputAndSerializationMatchChild(same);
+    }
+  );
+
+  // Cleanup
+  BrowserTestUtils.removeTab(tab);
+  Services.cookies.removeAll();
+});

netwerk/test/browser/browser_simple_unknown_uris_sync.js

@@ -0,0 +1,152 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+ChromeUtils.defineESModuleGetters(this, {
+  RemoteSettings: "resource://services-settings/remote-settings.sys.mjs",
+});
+
+const {
+  checkInputAndSerializationMatch,
+  checkInputAndSerializationMatchChild,
+  checkSerializationMissingSecondColon,
+  checkSerializationMissingSecondColonChild,
+  removeSecondColon,
+  runParentTestSuite,
+} = ChromeUtils.importESModule(
+  "resource://testing-common/simple_unknown_uri_helpers.sys.mjs"
+);
+
+const { sinon } = ChromeUtils.importESModule(
+  "resource://testing-common/Sinon.sys.mjs"
+);
+
+add_setup(async function () {
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      ["network.url.useDefaultURI", true],
+      ["network.url.simple_uri_unknown_schemes_enabled", true],
+      ["network.url.simple_uri_unknown_schemes", "simpleprotocol,otherproto"],
+    ],
+  });
+});
+
+const bypassCollectionName = "url-parser-default-unknown-schemes-interventions";
+
+let newData = [
+  {
+    id: "111",
+    scheme: "testinitscheme",
+  },
+  {
+    id: "112",
+    scheme: "testsyncscheme",
+  },
+];
+
+// sync update, test on parent
+add_task(async function test_bypass_list_update_sync_parent() {
+  const settings = await RemoteSettings(bypassCollectionName);
+  let stub = sinon.stub(settings, "get").returns(newData);
+  registerCleanupFunction(async function () {
+    stub.restore();
+  });
+
+  await RemoteSettings(bypassCollectionName).emit("sync", {});
+
+  runParentTestSuite();
+
+  stub.restore();
+});
+
+// sync update, test on child
+add_task(async function test_bypass_list_update_sync_child() {
+  const settings = await RemoteSettings(bypassCollectionName);
+  let stub = sinon.stub(settings, "get").returns(newData);
+  registerCleanupFunction(async function () {
+    stub.restore();
+  });
+
+  const URL_EXAMPLE = "https://example.com";
+  const tab = BrowserTestUtils.addTab(gBrowser, URL_EXAMPLE);
+  const browser = gBrowser.getBrowserForTab(tab);
+  await BrowserTestUtils.browserLoaded(browser);
+
+  await RemoteSettings(bypassCollectionName).emit("sync", {});
+
+  await SpecialPowers.spawn(
+    browser,
+    [
+      removeSecondColon.toString(),
+      checkSerializationMissingSecondColonChild.toString(),
+      checkInputAndSerializationMatchChild.toString(),
+    ],
+    (rscSource, csmscSource, ciasmcSource) => {
+      /* eslint-disable no-eval */
+      // eslint-disable-next-line no-unused-vars
+      let removeSecondColon = eval(`(${rscSource})`); // used by checker fns
+      let checkSerializationMissingSecondColonChild = eval(`(${csmscSource})`);
+      let checkInputAndSerializationMatchChild = eval(`(${ciasmcSource})`);
+      /* eslint-enable no-eval */
+
+      // sanity check
+      checkInputAndSerializationMatchChild("https://example.com/");
+
+      // nsStanardURL
+      checkSerializationMissingSecondColonChild("https://https://example.com");
+
+      // no-bypass protocol uses defaultURI
+      checkSerializationMissingSecondColonChild(
+        "defaulturischeme://https://example.com"
+      );
+
+      // an unknown protocol in the bypass list (remote settings) uses simpleURI
+      checkInputAndSerializationMatchChild(
+        "testsyncscheme://https://example.com"
+      );
+
+      // pref-specified scheme bypass uses simpleURI
+      checkInputAndSerializationMatchChild(
+        "simpleprotocol://https://example.com"
+      );
+    }
+  );
+
+  // Cleanup
+  stub.restore();
+  BrowserTestUtils.removeTab(tab);
+});
+
+// long string
+add_task(async function test_bypass_list_update_sync_parent_long_string() {
+  let longSchemeList = ["testinitscheme", "testsyncscheme"];
+  let num = 100;
+  for (let i = 0; i <= num; i++) {
+    longSchemeList.push(`scheme${i}`);
+  }
+
+  let newData = [];
+  for (const i in longSchemeList) {
+    newData.push({ id: i, scheme: longSchemeList[i] });
+  }
+
+  const settings = await RemoteSettings(bypassCollectionName);
+  let stub = sinon.stub(settings, "get").returns(newData);
+  registerCleanupFunction(async function () {
+    stub.restore();
+  });
+
+  await RemoteSettings(bypassCollectionName).emit("sync", {});
+
+  runParentTestSuite();
+
+  // another unknown protocol in the bypass list, near the middle of long str
+  checkInputAndSerializationMatch("scheme50://https://example.com");
+
+  // another unknown protocol in the bypass list, at the end of the long str
+  checkInputAndSerializationMatch("scheme100://https://example.com");
+
+  stub.restore();
+});

netwerk/test/browser/simple_unknown_uri_helpers.sys.mjs

@@ -0,0 +1,73 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { NetUtil } from "resource://gre/modules/NetUtil.sys.mjs";
+import { Assert } from "resource://testing-common/Assert.sys.mjs";
+
+export function checkInputAndSerializationMatch(input) {
+  let uri = NetUtil.newURI(input);
+  Assert.equal(
+    uri.spec,
+    input,
+    `serialization should match the input: {input}`
+  );
+}
+
+export function checkInputAndSerializationMatchChild(input) {
+  let uri = Services.io.newURI(input);
+  Assert.equal(
+    uri.spec,
+    input,
+    `serialization should match the input: {input}`
+  );
+}
+
+export function removeSecondColon(str) {
+  let colonIndex = str.indexOf(":");
+  if (colonIndex !== -1) {
+    colonIndex = str.indexOf(":", colonIndex + 1);
+    if (colonIndex !== -1) {
+      return str.slice(0, colonIndex) + str.slice(colonIndex + 1);
+    }
+  }
+  Assert.ok(false, "we expected at least two colons");
+  return str;
+}
+
+export function checkSerializationMissingSecondColon(input) {
+  let uri = NetUtil.newURI(input);
+  Assert.equal(
+    uri.spec,
+    removeSecondColon(input),
+    `serialization should be missing second colon from input: {input}`
+  );
+}
+
+export function checkSerializationMissingSecondColonChild(input) {
+  let uri = Services.io.newURI(input);
+  Assert.equal(
+    uri.spec,
+    removeSecondColon(input),
+    `serialization should be missing second colon from input: {input}`
+  );
+}
+
+export function runParentTestSuite() {
+  // sanity check
+  checkInputAndSerializationMatch("https://example.com/");
+
+  // special scheme uses nsStanardURL
+  checkSerializationMissingSecondColon("https://https://example.com");
+
+  // no-bypass protocol uses defaultURI
+  checkSerializationMissingSecondColon(
+    "defaulturischeme://https://example.com"
+  );
+
+  // an unknown protocol in the bypass list (remote settings) uses simpleURI
+  checkInputAndSerializationMatch("testsyncscheme://https://example.com");
+
+  // pref-specified scheme bypass uses simpleURI
+  checkInputAndSerializationMatch("simpleprotocol://https://example.com");
+}

netwerk/test/moz.build

@@ -25,6 +25,7 @@ XPCSHELL_TESTS_MANIFESTS += [
 TESTING_JS_MODULES += [
     "browser/cookie_filtering_helper.sys.mjs",
     "browser/early_hint_preload_test_helper.sys.mjs",
+    "browser/simple_unknown_uri_helpers.sys.mjs",
     "unit/test_http3_prio_helpers.js",
 ]
 

netwerk/test/unit/test_default_uri_bypass.js

@@ -1,67 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-/*
- *  Test that default uri is bypassable by an unknown protocol that is
- *  present in the bypass list (and the pref is enabled)
- */
-"use strict";
-
-function inChildProcess() {
-  return Services.appinfo.processType != Ci.nsIXULRuntime.PROCESS_TYPE_DEFAULT;
-}
-
-function run_test() {
-  // In-Parent-only process pref setup
-  if (!inChildProcess()) {
-    Services.prefs.setBoolPref("network.url.useDefaultURI", true);
-    Services.prefs.setBoolPref(
-      "network.url.some_schemes_bypass_defaultURI_fallback",
-      true
-    );
-
-    Services.prefs.setCharPref(
-      "network.url.simple_uri_schemes",
-      "simpleprotocol,otherproto"
-    );
-  }
-
-  // check valid url is fine
-  let uri = NetUtil.newURI("https://example.com/");
-  Assert.equal(uri.spec, "https://example.com/"); // same
-
-  // nsStandardURL removes second colon when nesting protocols
-  let uri1 = NetUtil.newURI("https://https://example.com/");
-  Assert.equal(uri1.spec, "https://https//example.com/");
-
-  // defaultUri removes second colon
-  // no-bypass protocol uses defaultURI
-  let uri2 = NetUtil.newURI("nonsimpleprotocol://https://example.com");
-  Assert.equal(uri2.spec, "nonsimpleprotocol://https//example.com");
-
-  // simpleURI keeps the second colon
-  // an unknown protocol in the bypass list will use simpleURI
-  // despite network.url.useDefaultURI being enabled
-  let same = "simpleprotocol://https://example.com";
-  let uri3 = NetUtil.newURI(same);
-  Assert.equal(uri3.spec, same); // simple uri keeps second colon
-
-  // setCharPref not accessible from child process
-  if (!inChildProcess()) {
-    // check that pref update removes simpleprotocol from bypass list
-    Services.prefs.setCharPref("network.url.simple_uri_schemes", "otherproto");
-    let uri4 = NetUtil.newURI("simpleprotocol://https://example.com");
-    Assert.equal(uri4.spec, "simpleprotocol://https//example.com"); // default uri removes second colon
-
-    // check that spaces are parsed out
-    Services.prefs.setCharPref(
-      "network.url.simple_uri_schemes",
-      " simpleprotocol , otherproto "
-    );
-    let uri5 = NetUtil.newURI("simpleprotocol://https://example.com");
-    Assert.equal(uri5.spec, "simpleprotocol://https://example.com"); // simple uri keeps second colon
-    let uri6 = NetUtil.newURI("otherproto://https://example.com");
-    Assert.equal(uri6.spec, "otherproto://https://example.com"); // simple uri keeps second colon
-  }
-}

netwerk/test/unit/test_simple_unknown_uris.js

@@ -0,0 +1,70 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ *  Test that default uri is bypassable by an unknown protocol that is
+ *  present in the bypass list (and the pref is enabled)
+ */
+"use strict";
+
+const {
+  checkInputAndSerializationMatch,
+  checkSerializationMissingSecondColon,
+} = ChromeUtils.importESModule(
+  "resource://testing-common/simple_unknown_uri_helpers.sys.mjs"
+);
+
+function inChildProcess() {
+  return Services.appinfo.processType != Ci.nsIXULRuntime.PROCESS_TYPE_DEFAULT;
+}
+
+function run_test() {
+  // In-Parent-only process pref setup
+  if (!inChildProcess()) {
+    // child-test sets these in test_simple_unknown_uris_wrap.js
+    Services.prefs.setBoolPref("network.url.useDefaultURI", true);
+    Services.prefs.setBoolPref(
+      "network.url.simple_uri_unknown_schemes_enabled",
+      true
+    );
+    Services.prefs.setCharPref(
+      "network.url.simple_uri_unknown_schemes",
+      "simpleprotocol,otherproto"
+    );
+  }
+
+  // sanity check: non-nested special url is fine
+  checkInputAndSerializationMatch("https://example.com/");
+
+  // nsStandardURL removes second colon when nesting protocols
+  checkSerializationMissingSecondColon("https://https://example.com/");
+
+  // no-bypass for unknown protocol uses defaultURI
+  checkSerializationMissingSecondColon(
+    "nonsimpleprotocol://https://example.com"
+  );
+
+  // an unknown protocol in the bypass list will use simpleURI
+  checkInputAndSerializationMatch("simpleprotocol://https://example.com");
+
+  // setCharPref not accessible from child process
+  if (!inChildProcess()) {
+    // check that pref update removes simpleprotocol from bypass list
+    Services.prefs.setCharPref(
+      "network.url.simple_uri_unknown_schemes",
+      "otherproto"
+    );
+    checkSerializationMissingSecondColon(
+      "simpleprotocol://https://example.com"
+    );
+
+    // check that spaces are parsed out
+    Services.prefs.setCharPref(
+      "network.url.simple_uri_unknown_schemes",
+      " simpleprotocol , otherproto "
+    );
+    checkInputAndSerializationMatch("simpleprotocol://https://example.com");
+    checkInputAndSerializationMatch("otherproto://https://example.com");
+  }
+}

netwerk/test/unit/xpcshell.toml

@@ -526,8 +526,6 @@ skip-if = ["os == 'android'"] # Bug 1700483
 
 ["test_defaultURI.js"]
 
-["test_default_uri_bypass.js"]
-
 ["test_dns_by_type_resolve.js"]
 
 ["test_dns_cancel.js"]
@@ -1110,6 +1108,8 @@ skip-if = ["os != 'win'"]
 
 ["test_simple.js"]
 
+["test_simple_unknown_uris.js"]
+
 ["test_sockettransportsvc_available.js"]
 
 ["test_socks.js"]

netwerk/test/unit_ipc/test_simple_unknown_uris_wrap.js

@@ -1,13 +1,13 @@
 function run_test() {
   Services.prefs.setBoolPref("network.url.useDefaultURI", true);
   Services.prefs.setBoolPref(
-    "network.url.some_schemes_bypass_defaultURI_fallback",
+    "network.url.simple_uri_unknown_schemes_enabled",
     true
   );
   Services.prefs.setCharPref(
-    "network.url.simple_uri_schemes",
+    "network.url.simple_uri_unknown_schemes",
     "simpleprotocol,otherproto"
   );
 
-  run_test_in_child("../unit/test_default_uri_bypass.js");
+  run_test_in_child("../unit/test_simple_unknown_uris.js");
 }

netwerk/test/unit_ipc/xpcshell.toml

@@ -70,7 +70,7 @@ support-files = [
   "!/netwerk/test/unit/test_http3_prio_helpers.js",
   "!/netwerk/test/unit/http2-ca.pem",
   "!/netwerk/test/unit/test_orb_empty_header.js",
-  "!/netwerk/test/unit/test_default_uri_bypass.js",
+  "!/netwerk/test/unit/test_simple_unknown_uris.js",
   "child_is_proxy_used.js",
   "child_cookie_header.js",
   "child_dns_by_type_resolve.js",
@@ -226,6 +226,9 @@ prefs = ["network.allow_raw_sockets_in_content_processes=true"]
 ["test_resumable_channel_wrap.js"]
 prefs = ["network.allow_raw_sockets_in_content_processes=true"]
 
+["test_simple_unknown_uris_wrap.js"]
+prefs = ["network.allow_raw_sockets_in_content_processes=true"]
+
 ["test_simple_wrap.js"]
 prefs = ["network.allow_raw_sockets_in_content_processes=true"]
 
@@ -240,6 +243,3 @@ skip-if = ["os == 'android'"]
 
 ["test_xmlhttprequest_wrap.js"]
 prefs = ["network.allow_raw_sockets_in_content_processes=true"]
-
-["test_default_uri_bypass_wrap.js"]
-prefs = ["network.allow_raw_sockets_in_content_processes=true"]

services/settings/dumps/main/moz.build

@@ -30,6 +30,7 @@ if not CONFIG["MOZ_BUILD_APP"].startswith("mobile/"):
         "translations-models.json",
         "translations-wasm.json",
         "url-classifier-skip-urls.json",
+        "url-parser-default-unknown-schemes-interventions.json",
         "websites-with-shared-credential-backends.json",
     ]
 

services/settings/dumps/main/url-parser-default-unknown-schemes-interventions.json

@@ -0,0 +1,35 @@
+{
+  "data": [
+    {
+      "schema": 1726749097244,
+      "scheme": "ed2k",
+      "id": "2e569b4e-b4ff-4732-8b22-7bc2d68de052",
+      "last_modified": 1726769128879
+    },
+    {
+      "schema": 1726749132565,
+      "scheme": "openimstoolkit",
+      "id": "d4c08df9-f28a-400e-9788-1104a7076e8f",
+      "last_modified": 1726769128877
+    },
+    {
+      "schema": 1726749132850,
+      "scheme": "archipelago",
+      "id": "5581aab3-4eb9-44a7-9bde-fa38fdc2ff8d",
+      "last_modified": 1726769128874
+    },
+    {
+      "schema": 1726749133141,
+      "scheme": "winscp-scp",
+      "id": "536286bd-bb86-4549-8c7e-b5df8bb067cc",
+      "last_modified": 1726769128871
+    },
+    {
+      "schema": 1726749133424,
+      "scheme": "vscode",
+      "id": "7a89397c-722f-4603-b210-8de09905616a",
+      "last_modified": 1726769128869
+    }
+  ],
+  "timestamp": 1726769128879
+}