Backed out 2 changesets (bug 1736658) as per request from dev. CLOSED TREE

Backed out changeset d7414af6a093 (bug 1736658)
Backed out changeset a0f2619a6ba3 (bug 1736658)

browser/components/search/SearchSERPTelemetry.jsm

@@ -24,6 +24,8 @@ const SEARCH_WITH_ADS_SCALAR_OLD = "browser.search.with_ads";
 const SEARCH_WITH_ADS_SCALAR_BASE = "browser.search.withads.";
 const SEARCH_AD_CLICKS_SCALAR_OLD = "browser.search.ad_clicks";
 const SEARCH_AD_CLICKS_SCALAR_BASE = "browser.search.adclicks.";
+const SEARCH_DATA_TRANSFERRED_SCALAR = "browser.search.data_transferred";
+const SEARCH_TELEMETRY_PRIVATE_BROWSING_KEY_SUFFIX = "pb";
 
 const TELEMETRY_SETTINGS_KEY = "search-telemetry";
 
@@ -596,6 +598,8 @@ class ContentHandler {
     Cc["@mozilla.org/network/http-activity-distributor;1"]
       .getService(Ci.nsIHttpActivityDistributor)
       .addObserver(this);
+
+    Services.obs.addObserver(this, "http-on-stop-request");
   }
 
   /**
@@ -605,6 +609,8 @@ class ContentHandler {
     Cc["@mozilla.org/network/http-activity-distributor;1"]
       .getService(Ci.nsIHttpActivityDistributor)
       .removeObserver(this);
+
+    Services.obs.removeObserver(this, "http-on-stop-request");
   }
 
   /**
@@ -617,6 +623,85 @@ class ContentHandler {
     Services.ppmm.sharedData.set("SearchTelemetry:ProviderInfo", providerInfo);
   }
 
+  /**
+   * Reports bandwidth used by the given channel if it is used by search requests.
+   *
+   * @param {object} aChannel The channel that generated the activity.
+   */
+  _reportChannelBandwidth(aChannel) {
+    if (!(aChannel instanceof Ci.nsIChannel)) {
+      return;
+    }
+    let wrappedChannel = ChannelWrapper.get(aChannel);
+
+    let getTopURL = channel => {
+      // top-level document
+      if (
+        channel.loadInfo &&
+        channel.loadInfo.externalContentPolicyType ==
+          Ci.nsIContentPolicy.TYPE_DOCUMENT
+      ) {
+        return channel.finalURL;
+      }
+
+      // iframe
+      let frameAncestors;
+      try {
+        frameAncestors = channel.frameAncestors;
+      } catch (e) {
+        frameAncestors = null;
+      }
+      if (frameAncestors) {
+        let ancestor = frameAncestors.find(obj => obj.frameId == 0);
+        if (ancestor) {
+          return ancestor.url;
+        }
+      }
+
+      // top-level resource
+      if (channel.loadInfo && channel.loadInfo.loadingPrincipal) {
+        return channel.loadInfo.loadingPrincipal.spec;
+      }
+
+      return null;
+    };
+
+    let topUrl = getTopURL(wrappedChannel);
+    if (!topUrl) {
+      return;
+    }
+
+    let info = this._checkURLForSerpMatch(topUrl);
+    if (!info) {
+      return;
+    }
+
+    let bytesTransferred =
+      wrappedChannel.requestSize + wrappedChannel.responseSize;
+    let { provider } = info;
+
+    let isPrivate =
+      wrappedChannel.loadInfo &&
+      wrappedChannel.loadInfo.originAttributes.privateBrowsingId > 0;
+    if (isPrivate) {
+      provider += `-${SEARCH_TELEMETRY_PRIVATE_BROWSING_KEY_SUFFIX}`;
+    }
+
+    Services.telemetry.keyedScalarAdd(
+      SEARCH_DATA_TRANSFERRED_SCALAR,
+      provider,
+      bytesTransferred
+    );
+  }
+
+  observe(aSubject, aTopic, aData) {
+    switch (aTopic) {
+      case "http-on-stop-request":
+        this._reportChannelBandwidth(aSubject);
+        break;
+    }
+  }
+
   /**
    * Listener that observes network activity, so that we can determine if a link
    * from a search provider page was followed, and if then if that link was an

browser/components/search/test/browser/browser_search_telemetry_sources_ads.js

@@ -84,6 +84,34 @@ add_task(async function test_simple_search_page_visit() {
   );
 });
 
+add_task(async function test_simple_search_page_visit_telemetry() {
+  searchCounts.clear();
+  Services.telemetry.clearScalars();
+
+  await BrowserTestUtils.withNewTab(
+    {
+      gBrowser,
+      /* URL must not be in the cache */
+      url: getSERPUrl(getPageUrl()) + `&random=${Math.random()}`,
+    },
+    async () => {
+      let scalars = {};
+      const key = "browser.search.data_transferred";
+
+      await TestUtils.waitForCondition(() => {
+        scalars =
+          Services.telemetry.getSnapshotForKeyedScalars("main", false).parent ||
+          {};
+        return key in scalars;
+      }, "should have the expected keyed scalars");
+
+      const scalar = scalars[key];
+      Assert.ok("example" in scalar, "correct telemetry category");
+      Assert.notEqual(scalars[key].example, 0, "bandwidth logged");
+    }
+  );
+});
+
 add_task(async function test_follow_on_visit() {
   await BrowserTestUtils.withNewTab(
     {

toolkit/components/search/SearchSuggestionController.jsm

@@ -22,6 +22,10 @@ const BROWSER_SUGGEST_PRIVATE_PREF = "browser.search.suggest.enabled.private";
 const REMOTE_TIMEOUT_PREF = "browser.search.suggest.timeout";
 const REMOTE_TIMEOUT_DEFAULT = 500; // maximum time (ms) to wait before giving up on a remote suggestions
 
+const SEARCH_DATA_TRANSFERRED_SCALAR = "browser.search.data_transferred";
+const SEARCH_TELEMETRY_KEY_PREFIX = "sggt";
+const SEARCH_TELEMETRY_PRIVATE_BROWSING_KEY_SUFFIX = "pb";
+
 /**
  * Generates an UUID.
  *
@@ -359,6 +363,36 @@ SearchSuggestionController.prototype = {
     });
   },
 
+  /**
+   * Report bandwidth used by search activities. It only reports when it matches
+   * search provider information.
+   *
+   * @param {string} engineId the name of the search provider.
+   * @param {boolean} privateMode set to true if this is coming from a private browsing mode request.
+   */
+  _reportBandwidthForEngine(engineId, privateMode) {
+    if (!this._request || !this._request.channel) {
+      return;
+    }
+
+    let channel = ChannelWrapper.get(this._request.channel);
+    let bytesTransferred = channel.requestSize + channel.responseSize;
+    if (bytesTransferred == 0) {
+      return;
+    }
+
+    let telemetryKey = `${SEARCH_TELEMETRY_KEY_PREFIX}-${engineId}`;
+    if (privateMode) {
+      telemetryKey += `-${SEARCH_TELEMETRY_PRIVATE_BROWSING_KEY_SUFFIX}`;
+    }
+
+    Services.telemetry.keyedScalarAdd(
+      SEARCH_DATA_TRANSFERRED_SCALAR,
+      telemetryKey,
+      bytesTransferred
+    );
+  },
+
   /**
    * Fetch suggestions from the search engine over the network.
    *
@@ -409,16 +443,20 @@ SearchSuggestionController.prototype = {
 
     this._request.mozBackgroundRequest = true; // suppress dialogs and fail silently
 
+    let engineId = engine.identifier || "other";
+
     this._request.addEventListener(
       "load",
-      this._onRemoteLoaded.bind(this, deferredResponse)
+      this._onRemoteLoaded.bind(this, deferredResponse, engineId, privateMode)
     );
     this._request.addEventListener("error", evt => {
+      this._reportBandwidthForEngine(engineId, privateMode);
       deferredResponse.resolve("HTTP error");
     });
     // Reject for an abort assuming it's always from .stop() in which case we shouldn't return local
     // or remote results for existing searches.
     this._request.addEventListener("abort", evt => {
+      this._reportBandwidthForEngine(engineId, privateMode);
       deferredResponse.reject("HTTP request aborted");
     });
 
@@ -437,9 +475,13 @@ SearchSuggestionController.prototype = {
    *
    * @param {Promise} deferredResponse
    *   The promise to resolve when a response is received.
+   * @param {string} engineId
+   *   The name of the search provider.
+   * @param {boolean} privateMode
+   *   Set to true if this is coming from a private browsing mode request.
    * @private
    */
-  _onRemoteLoaded(deferredResponse) {
+  _onRemoteLoaded(deferredResponse, engineId, privateMode) {
     if (!this._request) {
       deferredResponse.resolve(
         "Got HTTP response after the request was cancelled"
@@ -447,6 +489,8 @@ SearchSuggestionController.prototype = {
       return;
     }
 
+    this._reportBandwidthForEngine(engineId, privateMode);
+
     let status, serverResults;
     try {
       status = this._request.status;

toolkit/components/search/tests/xpcshell/test_searchSuggest.js

@@ -137,6 +137,26 @@ add_task(async function simple_remote_no_local_result() {
   Assert.equal(result.remote[2].value, "mom");
 });
 
+add_task(async function simple_remote_no_local_result_telemetry() {
+  Services.telemetry.clearScalars();
+
+  let controller = new SearchSuggestionController();
+  await controller.fetch("mo", false, getEngine);
+
+  let scalars = {};
+  const key = "browser.search.data_transferred";
+
+  await TestUtils.waitForCondition(() => {
+    scalars =
+      Services.telemetry.getSnapshotForKeyedScalars("main", false).parent || {};
+    return key in scalars;
+  }, "should have the expected keyed scalars");
+
+  const scalar = scalars[key];
+  Assert.ok("sggt-other" in scalar, "correct telemetry category");
+  Assert.notEqual(scalar["sggt-other"], 0, "bandwidth logged");
+});
+
 add_task(async function simple_remote_no_local_result_alternative_type() {
   let controller = new SearchSuggestionController();
   let result = await controller.fetch("mo", false, alternateJSONEngine);

toolkit/components/telemetry/Scalars.yaml

@@ -4903,6 +4903,31 @@ browser.search:
     record_in_processes:
       - main
 
+  data_transferred:
+    bug_numbers:
+      - 1614777
+      - 1619954
+      - 1656135
+      - 1694852
+    description: >
+      Records bytes that have been transferred for search requests, including
+      top-level documents, sub-resources, iframes and search suggestions.
+      The key format could be one of:
+        <search-provider-name>
+        <search-provider-name>-pb
+        sggt-<search-engine-id>
+        sggt-<search-engine-id>-pb
+    expires: "96"
+    keyed: true
+    kind: uint
+    notification_emails:
+      - tihuang@mozilla.com
+    products:
+      - 'firefox'
+    record_in_processes:
+      - 'main'
+    release_channel_collection: opt-out
+
 browser.search.content:
   urlbar:
     bug_numbers: