Bug 1943775 - Create a performance test for DoH TRRs r=sparky,necko-reviewers,perftest-reviewers,valentin

Using the browsertime custom test frame to validate DNS lookup times for popular TRRs and for native.
This is could give us an early awareness of TRR server issues.

Differential Revision: https://phabricator.services.mozilla.com/D235542

taskcluster/kinds/test/browsertime-desktop.yml

@@ -1199,3 +1199,27 @@ browsertime-indexeddb:
     mozharness:
         extra-options:
             - --extra-profiler-run
+
+browsertime-trr-performance:
+    raptor:
+        apps: [firefox]
+        run-visual-metrics: false
+        subtests:
+            - trr-cloudf
+            - trr-cira
+            - trr-comcas
+            - trr-next
+            - trr-native
+            - trr-w-clou
+            - trr-w-cira
+            - trr-w-comc
+            - trr-w-next
+
+    description: Raptor (browsertime) trr performance tests
+    max-run-time: 600
+    run-on-projects:
+        by-test-platform:
+            windows11-64-24h2-shippable/opt: [mozilla-central]
+            default: []
+    tier: 3
+    treeherder-symbol: Btime(trrperf)

taskcluster/kinds/test/test-sets.yml

@@ -111,6 +111,7 @@ browsertime:
     - browsertime-video-playback-latency
     - browsertime-network-bench
     - browsertime-webcodecs
+    - browsertime-trr-performance
 
 browsertime-clang:
     - browsertime-tp6

testing/raptor/browsertime/trr_performance.js

@@ -0,0 +1,53 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env node */
+
+const { logTest } = require("./utils/profiling");
+
+module.exports = logTest(
+  "dns lookup pageload",
+  async function (context, commands) {
+    context.log.info("Starting a pageload to measure DNS lookup time");
+
+    const testType = `${context.options.browsertime.test_type}`;
+    context.log.info("testType: " + testType);
+
+    const url = "https://httpstat.us/";
+
+    await commands.navigate("about:blank");
+
+    // Idle to allow for confirmation
+    await commands.wait.byTime(45000);
+
+    if (testType === "trr_warm") {
+      // Ensure the trr connection has been warmed up by making an arbitrary request
+      await commands.navigate("https://www.w3.org");
+      await commands.wait.byTime(2000);
+    }
+
+    // Start measuring
+    await commands.measure.start();
+    await commands.navigate(url);
+    await commands.measure.stop();
+
+    let dns_lookup_time = await commands.js.run(`
+      return (window.performance.timing.domainLookupEnd - window.performance.timing.domainLookupStart);
+    `);
+
+    let connect_time = await commands.js.run(`
+      return (window.performance.timing.connectEnd - window.performance.timing.navigationStart);
+    `);
+
+    context.log.info("dns_lookup_time: " + dns_lookup_time);
+    context.log.info("connect_time: " + connect_time);
+
+    await commands.measure.addObject({
+      custom_data: { dns_lookup_time, connect_time },
+    });
+
+    context.log.info("DNS lookup test finished.");
+    return true;
+  }
+);

testing/raptor/raptor/perfdocs/config.yml

@@ -238,6 +238,15 @@ suites:
             addkBLN: "Use add API to send a couple of large Blobs to IndexedDB independently"
             constant-regression: "Generates a constant value that can be changed to induce a regression."
             getkeyrng: "Measures the speed of key-range based reads of IndexedDB"
+            trr-cloudf: "Tests DNS lookup time on a Cloudflare TRR server."
+            trr-cira: "Tests DNS lookup time on a CIRA TRR server."
+            trr-comcas: "Tests DNS lookup time on a Comcast TRR server."
+            trr-next: "Tests DNS lookup time on a NextDNS TRR server."
+            trr-native: "Tests DNS lookup time via native DNS."
+            trr-w-clou: "Tests DNS lookup time on a Cloudflare TRR server, with the TRR connection warmed up."
+            trr-w-cira: "Tests DNS lookup time on a CIRA TRR server, with the TRR connection warmed up."
+            trr-w-comc: "Tests DNS lookup time on a Comcast TRR server, with the TRR connection warmed up."
+            trr-w-next: "Tests DNS lookup time on a NextDNS TRR server, with the TRR connection warmed up."
             upload: "Measures HTTP/2 file upload throughput with a remote server"
             upload-h3: "Measures HTTP/3 file upload throughput with a remote server"
             h3-upload: "Measures HTTP/3 file upload throughput with a local server"

testing/raptor/raptor/raptor.toml

@@ -5,10 +5,10 @@
 
 ["include:tests/benchmarks/assorted-dom.toml"]
 
-["include:tests/benchmarks/jetstream2.toml"]
-
 ["include:tests/benchmarks/jetstream2-mobile.toml"]
 
+["include:tests/benchmarks/jetstream2.toml"]
+
 ["include:tests/benchmarks/matrix-react-bench.toml"]
 
 ["include:tests/benchmarks/motionmark-1-3-desktop.toml"]
@@ -52,6 +52,9 @@
 ["include:tests/custom/browsertime-indexeddb.toml"]
 # Custom indexeddb test
 
+["include:tests/custom/browsertime-network-bench.toml"]
+# Custom download/upload performance test
+
 ["include:tests/custom/browsertime-process-switch.toml"]
 # Fission process switch time test
 
@@ -70,12 +73,12 @@
 ["include:tests/custom/browsertime-tp6-bench.toml"]
 # Pageload benchmark test used for getting an overview of impact
 
+["include:tests/custom/browsertime-trr-performance.toml"]
+# Custom trr performance tests
+
 ["include:tests/custom/browsertime-upload.toml"]
 # Custom upload speed test
 
-["include:tests/custom/browsertime-network-bench.toml"]
-# Custom download/upload performance test
-
 ["include:tests/custom/browsertime-video-playback-latency.toml"]
 # Video playback latency tests
 

testing/raptor/raptor/tests/custom/browsertime-trr-performance.toml

@@ -0,0 +1,77 @@
+[DEFAULT]
+apps = "firefox"
+cold = true
+gecko_profile_interval = 1
+lower_is_better = true
+owner = "Network Team"
+page_cycles = 1 # this is a cold pageload test
+page_timeout = 60000
+type = "pageload"
+unit = "ms"
+alert_threshold = 2.0
+use_live_sites = true
+
+# Keep this list in alphabetical order
+
+["trr-cloudf"]
+browsertime_args = "--browsertime.test_type=browser_idle --firefox.preference network.trr.mode:3 --firefox.preference network.trr.uri:https://mozilla.cloudflare-dns.com/dns-query"
+browser_cycles = 3
+test_script = "trr_performance.js"
+test_url = "None"
+custom_data = true
+
+["trr-cira"]
+browsertime_args = "--browsertime.test_type=browser_idle --firefox.preference network.trr.mode:3 --firefox.preference network.trr.uri:https://private.canadianshield.cira.ca/dns-query"
+browser_cycles = 3
+test_script = "trr_performance.js"
+test_url = "None"
+custom_data = true
+
+["trr-comcas"]
+browsertime_args = "--browsertime.test_type=browser_idle --firefox.preference network.trr.mode:3 --firefox.preference network.trr.uri:https://doh.xfinity.com/dns-query"
+browser_cycles = 3
+test_script = "trr_performance.js"
+test_url = "None"
+custom_data = true
+
+["trr-next"]
+browsertime_args = "--browsertime.test_type=browser_idle --firefox.preference network.trr.mode:3 --firefox.preference network.trr.uri:https://firefox.dns.nextdns.io/dns-query"
+browser_cycles = 3
+test_script = "trr_performance.js"
+test_url = "None"
+custom_data = true
+
+["trr-native"]
+browsertime_args = "--browsertime.test_type=browser_idle --firefox.preference network.trr.mode:0"
+browser_cycles = 3
+test_script = "trr_performance.js"
+test_url = "None"
+custom_data = true
+
+["trr-w-clou"]
+browsertime_args = "--browsertime.test_type=trr_warm --firefox.preference network.trr.mode:3 --firefox.preference network.trr.uri:https://mozilla.cloudflare-dns.com/dns-query"
+browser_cycles = 3
+test_script = "trr_performance.js"
+test_url = "None"
+custom_data = true
+
+["trr-w-cira"]
+browsertime_args = "--browsertime.test_type=trr_warm --firefox.preference network.trr.mode:3 --firefox.preference network.trr.uri:https://private.canadianshield.cira.ca/dns-query"
+browser_cycles = 3
+test_script = "trr_performance.js"
+test_url = "None"
+custom_data = true
+
+["trr-w-comc"]
+browsertime_args = "--browsertime.test_type=trr_warm --firefox.preference network.trr.mode:3 --firefox.preference network.trr.uri:https://doh.xfinity.com/dns-query"
+browser_cycles = 3
+test_script = "trr_performance.js"
+test_url = "None"
+custom_data = true
+
+["trr-w-next"]
+browsertime_args = "--browsertime.test_type=trr_warm --firefox.preference network.trr.mode:3 --firefox.preference network.trr.uri:https://firefox.dns.nextdns.io/dns-query"
+browser_cycles = 3
+test_script = "trr_performance.js"
+test_url = "None"
+custom_data = true