Bug 1956073 - Add Opt-in & additional telemetry for Smart Tab Grouping r=rrando,dwalker,tabbrowser-reviewers

- Telemetry for opt-in
     - For each onboarding step
- Telemetry updates for existing
     - Clicking away with ML group name without explicitly clicking "done" or "cancel"
     - When there are no suggestions for "suggest other tabs"
     - Add group id to ml telemetry
     - Move stg telemetry under tab group telemetry
     - Remove num_ from events

Differential Revision: https://phabricator.services.mozilla.com/D242784

browser/components/tabbrowser/SmartTabGrouping.sys.mjs

@@ -836,13 +836,20 @@ export class SmartTabGroupingManager {
    * @param {number} numTabsInGroup Number of tabs used to generate the label
    * @param {string} mlLabel ML generated label for the tab group
    * @param {string} userLabel User saved label for the tab group
+   * @param {string} id The id of the group
    */
-  async handleLabelTelemetry({ action, numTabsInGroup, mlLabel, userLabel }) {
+  async handleLabelTelemetry({
+    action,
+    numTabsInGroup,
+    mlLabel,
+    userLabel,
+    id = "",
+  }) {
     const { [ML_TASK_TEXT2TEXT]: topicEngineConfig } =
       await this.getEngineConfigs();
-    Glean.browserMlInteraction.smartTabTopic.record({
+    Glean.tabgroup.smartTabTopic.record({
       action,
-      num_tabs_in_group: numTabsInGroup,
+      tabs_in_group: numTabsInGroup,
       ml_label_length: (mlLabel || "").length,
       user_label_length: (userLabel || "").length,
       levenshtein_distance: lazy.NLP.levenshtein(
@@ -850,6 +857,7 @@ export class SmartTabGroupingManager {
         mlLabel || ""
       ),
       model_revision: topicEngineConfig.modelRevision || "",
+      id,
     });
   }
 
@@ -863,6 +871,7 @@ export class SmartTabGroupingManager {
    * @param {number} numTabsSuggested Number of tabs suggested by the model
    * @param {number} numTabsApproved Number of tabs approved by the user
    * @param {number} numTabsRemoved Number of tabs removed by the user
+   * @param {string} id The id of the group
    */
   async handleSuggestTelemetry({
     action,
@@ -871,17 +880,19 @@ export class SmartTabGroupingManager {
     numTabsSuggested,
     numTabsApproved,
     numTabsRemoved,
+    id = "",
   }) {
     const { [ML_TASK_FEATURE_EXTRACTION]: embeddingEngineConfig } =
       await this.getEngineConfigs();
-    Glean.browserMlInteraction.smartTabSuggest.record({
+    Glean.tabgroup.smartTabSuggest.record({
       action,
-      num_tabs_in_window: numTabsInWindow,
+      tabs_in_window: numTabsInWindow,
-      num_tabs_in_group: numTabsInGroup,
+      tabs_in_group: numTabsInGroup,
-      num_tabs_suggested: numTabsSuggested,
+      tabs_suggested: numTabsSuggested,
-      num_tabs_approved: numTabsApproved,
+      tabs_approved: numTabsApproved,
-      num_tabs_removed: numTabsRemoved,
+      tabs_removed: numTabsRemoved,
       model_revision: embeddingEngineConfig.modelRevision || "",
+      id,
     });
   }
 
@@ -893,14 +904,14 @@ export class SmartTabGroupingManager {
   async getEngineConfigs() {
     if (!this.topicEngineConfig) {
       this.topicEngineConfig = await lazy.MLEngineParent.getInferenceOptions(
-        this.config.topicGeneration.engineId,
+        this.config.topicGeneration.featureId,
         this.config.topicGeneration.taskName
       );
     }
     if (!this.embeddingEngineConfig) {
       this.embeddingEngineConfig =
         await lazy.MLEngineParent.getInferenceOptions(
-          this.config.embedding.engineId,
+          this.config.embedding.featureId,
           this.config.embedding.taskName
         );
     }

browser/components/tabbrowser/content/tabgroup-menu.js

@@ -474,6 +474,7 @@
     }
 
     #initSmartTabGroupsOptin() {
+      this.#handleMLOptinTelemetry("step0-optin-shown");
       this.suggestionState = MozTabbrowserTabGroupMenu.State.OPTIN;
 
       // Init optin component
@@ -486,19 +487,29 @@
 
       // On Confirm
       this.#suggestionsOptin.addEventListener("MlModelOptinConfirm", () => {
+        this.#handleMLOptinTelemetry("step1-optin-confirmed");
         Services.prefs.setBoolPref("browser.tabs.groups.smart.optin", true);
         this.#handleFirstDownloadAndSuggest();
       });
 
       // On Deny
       this.#suggestionsOptin.addEventListener("MlModelOptinDeny", () => {
-        this.SmartTabGroupingManager.terminateProcess();
+        this.#handleMLOptinTelemetry("step1-optin-denied");
+        this.#smartTabGroupingManager.terminateProcess();
         this.suggestionState = this.createMode
           ? MozTabbrowserTabGroupMenu.State.CREATE_AI_INITIAL
           : MozTabbrowserTabGroupMenu.State.EDIT_AI_INITIAL;
         this.#setFormToDisabled(false);
       });
 
+      // On Cancel Model Download
+      this.#suggestionsOptin.addEventListener(
+        "MlModelOptinCancelDownload",
+        () => {
+          this.#handleMLOptinTelemetry("step2-optin-cancel-download");
+        }
+      );
+
       this.#suggestionsOptinContainer.appendChild(this.#suggestionsOptin);
     }
 
@@ -806,6 +817,12 @@
           this.dispatchEvent(
             new CustomEvent("TabGroupCreateDone", { bubbles: true })
           );
+          if (
+            this.smartTabGroupsEnabled &&
+            (this.#suggestedMlLabel || this.#hasSuggestedMlTabs)
+          ) {
+            this.#handleMlTelemetry("save-popup-hidden");
+          }
         } else {
           this.activeGroup.ungroupTabs();
         }
@@ -940,6 +957,7 @@
       this.#suggestionsOptin.isHidden = true;
 
       // Continue on with the suggest flow
+      this.#handleMLOptinTelemetry("step3-optin-completed");
       this.#initMlGroupLabel();
       this.#handleSmartSuggest();
     }
@@ -957,6 +975,13 @@
         this.suggestionState = this.#createMode
           ? MozTabbrowserTabGroupMenu.State.CREATE_AI_WITH_NO_SUGGESTIONS
           : MozTabbrowserTabGroupMenu.State.EDIT_AI_WITH_NO_SUGGESTIONS;
+
+        // there's no "save" button from the edit ai interaction with
+        // no tab suggestions, so we need to capture here
+        if (!this.#createMode) {
+          this.#hasSuggestedMlTabs = true;
+          this.#handleMlTelemetry("save");
+        }
         return;
       }
 
@@ -975,7 +1000,7 @@
 
     /**
      * Sends Glean metrics if smart tab grouping is enabled
-     * @param {string} action "save" or "cancel"
+     * @param {string} action "save", "save-popup-hidden" or "cancel"
      */
     #handleMlTelemetry(action) {
       if (!this.smartTabGroupsEnabled) {
@@ -987,6 +1012,7 @@
           numTabsInGroup: this.#activeGroup.tabs.length,
           mlLabel: this.#suggestedMlLabel,
           userLabel: this.#nameField.value,
+          id: this.#activeGroup.id,
         });
         this.#suggestedMlLabel = "";
       }
@@ -999,11 +1025,22 @@
           numTabsApproved: this.#selectedSuggestedTabs.length,
           numTabsRemoved:
             this.#suggestedTabs.length - this.#selectedSuggestedTabs.length,
+          id: this.#activeGroup.id,
         });
         this.#hasSuggestedMlTabs = false;
       }
     }
 
+    /**
+     * Sends Glean metrics for opt-in UI flow
+     * @param {string} step contains step number and description of flow
+     */
+    #handleMLOptinTelemetry(step) {
+      Glean.tabgroup.smartTabOptin.record({
+        step,
+      });
+    }
+
     #createRow(tab, index) {
       // Create Row
       let row = document.createXULElement("toolbaritem");

browser/components/tabbrowser/metrics.yaml

@@ -119,91 +119,6 @@ browser.ui.interaction:
     no_lint:
       - COMMON_PREFIX
 
-browser.ml.interaction:
-  smart_tab_topic:
-    type: event
-    description: >
-      Recorded when the user saves a tab group label with the ml topic
-      model
-    notification_emails:
-      - rrando@mozilla.com
-      - vbaungally@mozilla.com
-    bugs:
-      - https://bugzil.la/1949010
-    data_reviews:
-      - https://bugzil.la/1949010
-    data_sensitivity:
-      - interaction
-    extra_keys:
-      action:
-        description: >
-          whether the user saved or canceled generating a label
-        type: string
-      num_tabs_in_group:
-        description: >
-          number of tabs in the group for which model is generating a
-          label
-        type: quantity
-      ml_label_length:
-        description: length of label suggested by model
-        type: quantity
-      user_label_length:
-        description: length of label saved by the user
-        type: quantity
-      levenshtein_distance:
-        description: >
-          Levenshtein distance between label suggested by model
-          and saved by user
-        type: quantity
-      model_revision:
-        description: topic model version currently running
-        type: string
-    expires: never
-    no_lint:
-      - COMMON_PREFIX
-
-  smart_tab_suggest:
-    type: event
-    description: >
-      Recorded when the user saves suggested tabs using the ml
-      embedding model
-    notification_emails:
-      - rrando@mozilla.com
-      - vbaungally@mozilla.com
-    bugs:
-      - https://bugzil.la/1949010
-    data_reviews:
-      - https://bugzil.la/1949010
-    data_sensitivity:
-      - interaction
-    extra_keys:
-      action:
-        description: whether the user saved or canceled
-        type: string
-      num_tabs_in_window:
-        description: number of tabs in current window
-        type: quantity
-      num_tabs_in_group:
-        description: number of tabs in the current group
-        type: quantity
-      num_tabs_suggested:
-        description: number of tabs suggested by the ml model
-        type: quantity
-      num_tabs_approved:
-        description: number of tabs the user saved from the suggestions
-        type: quantity
-      num_tabs_removed:
-        description: >
-          number of tabs the user removed from the
-          suggestions
-        type: quantity
-      model_revision:
-        description: embedding model version currently running
-        type: string
-    expires: never
-    no_lint:
-      - COMMON_PREFIX
-
 tabgroup:
   create_group:
     type: event
@@ -318,6 +233,117 @@ tabgroup:
       - inside
       - outside
 
+  smart_tab_optin:
+    type: event
+    description: >
+      Recorded when onboards to smart tab grouping through opt-in UI
+    notification_emails:
+      - rrando@mozilla.com
+      - vbaungally@mozilla.com
+    bugs:
+      - https://bugzil.la/1956073
+    data_reviews:
+      - https://bugzil.la/1956073
+    data_sensitivity:
+      - interaction
+    extra_keys:
+      step:
+        description: step and description of opt-in screen interaction
+        type: string
+    expires: never
+    no_lint:
+      - COMMON_PREFIX
+
+  smart_tab_topic:
+    type: event
+    description: >
+      Recorded when the user saves a tab group label with the ml topic
+      model
+    notification_emails:
+      - rrando@mozilla.com
+      - vbaungally@mozilla.com
+    bugs:
+      - https://bugzil.la/1949010
+    data_reviews:
+      - https://bugzil.la/1949010
+    data_sensitivity:
+      - interaction
+    extra_keys:
+      action:
+        description: >
+          whether the user saved or canceled generating a label
+        type: string
+      tabs_in_group:
+        description: >
+          number of tabs in the group for which model is generating a
+          label
+        type: quantity
+      ml_label_length:
+        description: length of label suggested by model
+        type: quantity
+      user_label_length:
+        description: length of label saved by the user
+        type: quantity
+      levenshtein_distance:
+        description: >
+          Levenshtein distance between label suggested by model
+          and saved by user
+        type: quantity
+      model_revision:
+        description: topic model version currently running
+        type: string
+      id:
+        description: The ID of the created tab group
+        type: string
+    expires: never
+    no_lint:
+      - COMMON_PREFIX
+
+  smart_tab_suggest:
+    type: event
+    description: >
+      Recorded when the user saves suggested tabs using the ml
+      embedding model
+    notification_emails:
+      - rrando@mozilla.com
+      - vbaungally@mozilla.com
+    bugs:
+      - https://bugzil.la/1949010
+    data_reviews:
+      - https://bugzil.la/1949010
+    data_sensitivity:
+      - interaction
+    extra_keys:
+      action:
+        description: whether the user saved or canceled
+        type: string
+      tabs_in_window:
+        description: number of tabs in current window
+        type: quantity
+      tabs_in_group:
+        description: number of tabs in the current group
+        type: quantity
+      tabs_suggested:
+        description: number of tabs suggested by the ml model
+        type: quantity
+      tabs_approved:
+        description: number of tabs the user saved from the suggestions
+        type: quantity
+      tabs_removed:
+        description: >
+          number of tabs the user removed from the
+          suggestions
+        type: quantity
+      model_revision:
+        description: embedding model version currently running
+        type: string
+      id:
+        description: The ID of the created tab group
+        type: string
+    expires: never
+    no_lint:
+      - COMMON_PREFIX
+
 browser.tabswitch:
   update:
     type: timing_distribution

browser/components/tabbrowser/test/browser/smarttabgrouping/browser_tab_grouping_telemetry.js

@@ -15,12 +15,12 @@ async function openCreatePanel(tabgroupPanel, tab) {
   await panelShown;
 }
 
-async function setup(enableSmartTab = true) {
+async function setup(enableSmartTab = true, optIn = true) {
   await SpecialPowers.pushPrefEnv({
     set: [
       ["browser.tabs.groups.enabled", true],
       ["browser.tabs.groups.smart.enabled", enableSmartTab],
-      ["browser.tabs.groups.smart.optin", true],
+      ["browser.tabs.groups.smart.optin", optIn],
     ],
   });
   sinon
@@ -33,6 +33,9 @@ async function setup(enableSmartTab = true) {
     "text2text-generation": { modelRevision: "v0.3.4" },
     "feature-extraction": { modelRevision: "v0.1.0" },
   });
+  sinon
+    .stub(SmartTabGroupingManager.prototype, "preloadAllModels")
+    .resolves([]);
 
   let tab = BrowserTestUtils.addTab(gBrowser, "about:blank");
   const cleanup = () => {
@@ -57,14 +60,10 @@ add_task(async function test_saving_ml_suggested_label_telemetry() {
   let panelHidden = BrowserTestUtils.waitForPopupEvent(tabgroupPanel, "hidden");
   await panelHidden;
 
-  const events = Glean.browserMlInteraction.smartTabTopic.testGetValue();
+  const events = Glean.tabgroup.smartTabTopic.testGetValue();
   Assert.equal(events.length, 1, "Should create a label event");
   Assert.equal(events[0].extra.action, "save", "Save button was clicked");
-  Assert.equal(
+  Assert.equal(events[0].extra.tabs_in_group, "1", "Number of tabs in group");
-    events[0].extra.num_tabs_in_group,
-    "1",
-    "Number of tabs in group"
-  );
   Assert.equal(events[0].extra.ml_label_length, "25", "Suggested ML Label");
   Assert.equal(
     events[0].extra.user_label_length,
@@ -93,18 +92,14 @@ add_task(async function test_cancel_ml_suggested_label_telemetry() {
   let panelHidden = BrowserTestUtils.waitForPopupEvent(tabgroupPanel, "hidden");
   await panelHidden;
 
-  const events = Glean.browserMlInteraction.smartTabTopic.testGetValue();
+  const events = Glean.tabgroup.smartTabTopic.testGetValue();
   Assert.equal(events.length, 1, "Should create a label event");
   Assert.equal(
     events[0].extra.action,
     "cancel",
     "Should save label even if cancel button was clicked"
   );
-  Assert.equal(
+  Assert.equal(events[0].extra.tabs_in_group, "1", "Number of tabs in group");
-    events[0].extra.num_tabs_in_group,
-    "1",
-    "Number of tabs in group"
-  );
   Assert.equal(events[0].extra.ml_label_length, "25", "Suggested ML Label");
   Assert.equal(
     events[0].extra.user_label_length,
@@ -135,26 +130,18 @@ add_task(async function test_saving_ml_suggested_tabs() {
   tabgroupPanel.querySelector("#tab-group-create-suggestions-button").click();
   let panelHidden = BrowserTestUtils.waitForPopupEvent(tabgroupPanel, "hidden");
   await panelHidden;
-  const events = Glean.browserMlInteraction.smartTabSuggest.testGetValue();
+  const events = Glean.tabgroup.smartTabSuggest.testGetValue();
   Assert.equal(events.length, 1, "Should create a sugggest event");
   Assert.equal(events[0].extra.action, "save", "Save button was clicked");
+  Assert.equal(events[0].extra.tabs_in_window, "2", "Number of tabs in window");
   Assert.equal(
-    events[0].extra.num_tabs_in_window,
+    events[0].extra.tabs_in_group,
-    "2",
-    "Number of tabs in window"
-  );
-  Assert.equal(
-    events[0].extra.num_tabs_in_group,
     "1",
     "Number of tabs in the group"
   );
-  Assert.equal(
+  Assert.equal(events[0].extra.tabs_suggested, "0", "No tabs were suggested");
-    events[0].extra.num_tabs_suggested,
+  Assert.equal(events[0].extra.tabs_approved, "0", "No tabs were approved");
-    "0",
+  Assert.equal(events[0].extra.tabs_removed, "0", "No tabs were removed");
-    "No tabs were suggested"
-  );
-  Assert.equal(events[0].extra.num_tabs_approved, "0", "No tabs were approved");
-  Assert.equal(events[0].extra.num_tabs_removed, "0", "No tabs were removed");
   Assert.equal(
     events[0].extra.model_revision,
     "v0.1.0",
@@ -181,13 +168,13 @@ add_task(async function test_saving_ml_suggested_tabs_with_ml_label() {
   let panelHidden = BrowserTestUtils.waitForPopupEvent(tabgroupPanel, "hidden");
   await panelHidden;
 
-  const labelEvent = Glean.browserMlInteraction.smartTabTopic.testGetValue();
+  const labelEvent = Glean.tabgroup.smartTabTopic.testGetValue();
-  const suggestEvent =
+  const suggestEvent = Glean.tabgroup.smartTabSuggest.testGetValue();
-    Glean.browserMlInteraction.smartTabSuggest.testGetValue();
   Assert.equal(labelEvent.length, 1, "Should create label event");
   Assert.equal(suggestEvent.length, 1, "Should create suggest event");
   Assert.equal(labelEvent[0].extra.action, "save", "Save button was clicked");
   Assert.equal(suggestEvent[0].extra.action, "save", "Save button was clicked");
+  Assert.ok(suggestEvent[0].extra.id, "Id of group should be present");
   cleanup();
 });
 
@@ -209,9 +196,8 @@ add_task(async function test_canceling_ml_suggested_tabs_with_ml_label() {
   let panelHidden = BrowserTestUtils.waitForPopupEvent(tabgroupPanel, "hidden");
   await panelHidden;
 
-  const labelEvent = Glean.browserMlInteraction.smartTabTopic.testGetValue();
+  const labelEvent = Glean.tabgroup.smartTabTopic.testGetValue();
-  const suggestEvent =
+  const suggestEvent = Glean.tabgroup.smartTabSuggest.testGetValue();
-    Glean.browserMlInteraction.smartTabSuggest.testGetValue();
   Assert.equal(labelEvent.length, 1, "Should create label event");
   Assert.equal(suggestEvent.length, 1, "Should create suggest event");
   Assert.equal(
@@ -224,6 +210,7 @@ add_task(async function test_canceling_ml_suggested_tabs_with_ml_label() {
     "cancel",
     "cancel button was clicked"
   );
+  Assert.ok(suggestEvent[0].extra.id, "Id of group should be present");
   cleanup();
 });
 
@@ -243,9 +230,89 @@ add_task(async function test_pref_off_should_not_create_events() {
   await panelHidden;
 
   Assert.equal(
-    Glean.browserMlInteraction.smartTabTopic.testGetValue() ?? "none",
+    Glean.tabgroup.smartTabTopic.testGetValue() ?? "none",
     "none",
     "No event if the feature is off"
   );
   cleanup();
 });
+
+add_task(async function test_saving_ml_label_popup_hidden_no_button_click() {
+  let { tab, cleanup } = await setup();
+  let tabgroupEditor = document.getElementById("tab-group-editor");
+  let tabgroupPanel = tabgroupEditor.panel;
+  let nameField = tabgroupPanel.querySelector("#tab-group-name");
+
+  await openCreatePanel(tabgroupPanel, tab);
+  // add the label
+  nameField.focus();
+  nameField.value = "Random ML Suggested Label"; // user label matching suggested label
+  tabgroupEditor.mlLabel = "Random ML Suggested Label"; // suggested label
+  // click on the suggest button
+  tabgroupPanel.querySelector("#tab-group-suggestion-button").click();
+  tabgroupEditor.hasSuggestedMlTabs = true;
+  tabgroupPanel.hidePopup();
+  let panelHidden = BrowserTestUtils.waitForPopupEvent(tabgroupPanel, "hidden");
+  await panelHidden;
+
+  const labelEvent = Glean.tabgroup.smartTabTopic.testGetValue();
+  const suggestEvent = Glean.tabgroup.smartTabSuggest.testGetValue();
+  Assert.equal(labelEvent.length, 1, "Should create label event");
+  Assert.equal(suggestEvent.length, 1, "Should create suggest event");
+  Assert.equal(
+    labelEvent[0].extra.action,
+    "save-popup-hidden",
+    "Popup was hidden by clicking away"
+  );
+  Assert.equal(
+    suggestEvent[0].extra.action,
+    "save-popup-hidden",
+    "Popup was hidden by clicking away"
+  );
+  Assert.ok(suggestEvent[0].extra.id, "Id of group should be present");
+  cleanup();
+});
+
+async function waitForUpdateComplete(element) {
+  if (element && typeof element.updateComplete === "object") {
+    await element.updateComplete;
+  }
+}
+
+add_task(async function test_optin_telemetry() {
+  let { tab, cleanup } = await setup(true, false);
+  let tabgroupEditor = document.getElementById("tab-group-editor");
+  let tabgroupPanel = tabgroupEditor.panel;
+
+  await openCreatePanel(tabgroupPanel, tab);
+  // click on suggestions flow
+  tabgroupPanel.querySelector("#tab-group-suggestion-button").click();
+  let optInEvent = Glean.tabgroup.smartTabOptin.testGetValue();
+  Assert.equal(
+    optInEvent.length,
+    1,
+    "Should create optin event for onboarding"
+  );
+  Assert.equal(
+    optInEvent[0].extra.step,
+    "step0-optin-shown",
+    "Should show proper step and description"
+  );
+
+  let mo = document.querySelector("model-optin");
+  Assert.ok(mo, "Found the ModelOptin element in the DOM.");
+  await waitForUpdateComplete(mo);
+
+  // first cancel the flow
+  Services.fog.testResetFOG();
+  const denyBtn = mo.shadowRoot.querySelector("#optin-deny-button");
+  denyBtn.click();
+  optInEvent = Glean.tabgroup.smartTabOptin.testGetValue();
+  Assert.equal(optInEvent.length, 1, "Should create optin cancel event");
+  Assert.equal(
+    optInEvent[0].extra.step,
+    "step1-optin-denied",
+    "Should show proper step and description"
+  );
+  cleanup();
+});