Bug 1817481 - define and record Nimbus enrollment_status metric and define Nimbus telemetry feature. r=barret

Differential Revision: https://phabricator.services.mozilla.com/D201642

toolkit/components/nimbus/FeatureManifest.yaml

@@ -2372,3 +2372,14 @@ nimbusIsReady:
     eventCount:
       description: The number of events that should be sent.
       type: int
+
+nimbusTelemetry:
+  description: A feature that enables or disables Nimbus telemetry.
+  owner: chumphreys@mozilla.com
+  hasExposure: false
+  applications:
+    - firefox-desktop
+  variables:
+    gleanMetricConfiguration:
+      description: A Glean metric configuration JSON blob.
+      type: json

toolkit/components/nimbus/lib/ExperimentManager.sys.mjs

@@ -32,6 +32,25 @@ const STUDIES_OPT_OUT_PREF = "app.shield.optoutstudies.enabled";
 
 const STUDIES_ENABLED_CHANGED = "nimbus:studies-enabled-changed";
 
+const ENROLLMENT_STATUS = {
+  ENROLLED: "Enrolled",
+  NOT_ENROLLED: "NotEnrolled",
+  DISQUALIFIED: "Disqualified",
+  WAS_ENROLLED: "WasEnrolled",
+  ERROR: "Error",
+};
+
+const ENROLLMENT_STATUS_REASONS = {
+  QUALIFIED: "Qualified",
+  OPT_IN: "OptIn",
+  OPT_OUT: "OptOut",
+  NOT_SELECTED: "NotSelected",
+  NOT_TARGETED: "NotTargeted",
+  ENROLLMENTS_PAUSED: "EnrollmentsPaused",
+  FEATURE_CONFLICT: "FeatureConflict",
+  ERROR: "Error",
+};
+
 function featuresCompat(branch) {
   if (!branch || (!branch.feature && !branch.features)) {
     return [];
@@ -182,6 +201,14 @@ export class _ExperimentManager {
     }
 
     this.observe();
+
+    lazy.NimbusFeatures.nimbusTelemetry.onUpdate(() => {
+      const cfg =
+        lazy.NimbusFeatures.nimbusTelemetry.getVariable(
+          "gleanMetricConfiguration"
+        ) ?? {};
+      Services.fog.setMetricsFeatureConfig(JSON.stringify(cfg));
+    });
   }
 
   /**
@@ -223,16 +250,22 @@ export class _ExperimentManager {
     missingL10nIds
   ) {
     for (const enrollment of enrollments) {
-      const { slug, source } = enrollment;
+      const { slug, source, branch } = enrollment;
       if (sourceToCheck !== source) {
         continue;
       }
+      const statusTelemetry = {
+        slug,
+        branch: branch.slug,
+      };
       if (!this.sessions.get(source)?.has(slug)) {
         lazy.log.debug(`Stopping study for recipe ${slug}`);
         try {
           let reason;
           if (recipeMismatches.includes(slug)) {
             reason = "targeting-mismatch";
+            statusTelemetry.status = ENROLLMENT_STATUS.DISQUALIFIED;
+            statusTelemetry.reason = ENROLLMENT_STATUS_REASONS.NOT_TARGETED;
           } else if (invalidRecipes.includes(slug)) {
             reason = "invalid-recipe";
           } else if (invalidBranches.has(slug) || invalidFeatures.has(slug)) {
@@ -243,12 +276,23 @@ export class _ExperimentManager {
             reason = "l10n-missing-entry";
           } else {
             reason = "recipe-not-seen";
+            statusTelemetry.status = ENROLLMENT_STATUS.WAS_ENROLLED;
+            statusTelemetry.branch = branch.slug;
+          }
+          if (!statusTelemetry.status) {
+            statusTelemetry.status = ENROLLMENT_STATUS.DISQUALIFIED;
+            statusTelemetry.reason = ENROLLMENT_STATUS_REASONS.ERROR;
+            statusTelemetry.error_string = reason;
           }
           this.unenroll(slug, reason);
         } catch (err) {
           console.error(err);
         }
+      } else {
+        statusTelemetry.status = ENROLLMENT_STATUS.ENROLLED;
+        statusTelemetry.reason = ENROLLMENT_STATUS_REASONS.QUALIFIED;
       }
+      this.sendEnrollmentStatusTelemetry(statusTelemetry);
     }
   }
 
@@ -755,6 +799,34 @@ export class _ExperimentManager {
     });
   }
 
+  /**
+   *
+   * @param {object} enrollmentStatus
+   * @param {string} enrollmentStatus.slug
+   * @param {string} enrollmentStatus.status
+   * @param {string?} enrollmentStatus.reason
+   * @param {string?} enrollmentStatus.branch
+   * @param {string?} enrollmentStatus.error_string
+   * @param {string?} enrollmentStatus.conflict_slug
+   */
+  sendEnrollmentStatusTelemetry({
+    slug,
+    status,
+    reason,
+    branch,
+    error_string,
+    conflict_slug,
+  }) {
+    Glean.nimbusEvents.enrollmentStatus.record({
+      slug,
+      status,
+      reason,
+      branch,
+      error_string,
+      conflict_slug,
+    });
+  }
+
   /**
    * Sets Telemetry when activating an experiment.
    *

toolkit/components/nimbus/metrics.yaml

@@ -223,10 +223,46 @@ nimbus_events:
       An event sent when Nimbus is ready — sent upon completion of each update of the recipes.
     bugs:
       - https://bugzilla.mozilla.org/show_bug.cgi?id=1875510
-    data_reviews: []
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1875510
     data_sensitivity:
       - technical
     notification_emails:
       - chumphreys@mozilla.com
       - project-nimbus@mozilla.com
     expires: 180
+
+  enrollment_status:
+    type: event
+    description: >
+      Recorded for each enrollment status each time the SDK completes application of pending experiments.
+    extra_keys:
+      slug:
+        type: string
+        description: The slug/unique identifier of the experiment
+      status:
+        type: string
+        description: The status of this enrollment
+      reason:
+        type: string
+        description: The reason the client is in the noted status
+      branch:
+        type: string
+        description: The branch slug/identifier that was randomly chosen (if the client is enrolled)
+      error_string:
+        type: string
+        description: If the enrollment resulted in an error, the associated error string
+      conflict_slug:
+        type: string
+        description: If the enrollment hit a feature conflict, the slug of the conflicting experiment/rollout
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1817481
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1817481
+    data_sensitivity:
+      - technical
+    notification_emails:
+      - chumphreys@mozilla.com
+      - project-nimbus@mozilla.com
+    expires: never
+    disabled: true

toolkit/components/nimbus/test/unit/test_ExperimentManager_unenroll.js

@@ -6,6 +6,9 @@ const { TelemetryEvents } = ChromeUtils.importESModule(
 const { TelemetryEnvironment } = ChromeUtils.importESModule(
   "resource://gre/modules/TelemetryEnvironment.sys.mjs"
 );
+const { ExperimentAPI } = ChromeUtils.importESModule(
+  "resource://nimbus/ExperimentAPI.sys.mjs"
+);
 const STUDIES_OPT_OUT_PREF = "app.shield.optoutstudies.enabled";
 const UPLOAD_ENABLED_PREF = "datareporting.healthreport.uploadEnabled";
 
@@ -487,3 +490,112 @@ add_task(async function test_rollout_telemetry_events() {
   );
   globalSandbox.restore();
 });
+
+add_task(async function test_check_unseen_enrollments_telemetry_events() {
+  globalSandbox.restore();
+  const store = ExperimentFakes.store();
+  const manager = ExperimentFakes.manager(store);
+  const sandbox = sinon.createSandbox();
+  sandbox.stub(manager, "unenroll").returns();
+  sandbox.stub(ExperimentAPI, "_store").get(() => manager.store);
+  sandbox.stub(ExperimentAPI, "_manager").get(() => manager);
+
+  await manager.onStartup();
+  await manager.store.ready();
+
+  const experiment = ExperimentFakes.recipe("foo", {
+    branches: [
+      {
+        slug: "wsup",
+        ratio: 1,
+        features: [
+          {
+            featureId: "nimbusTelemetry",
+            value: {
+              gleanMetricConfiguration: {
+                "nimbus_events.enrollment_status": true,
+              },
+            },
+          },
+        ],
+      },
+    ],
+    bucketConfig: {
+      ...ExperimentFakes.recipe.bucketConfig,
+      count: 1000,
+    },
+  });
+
+  await manager.enroll(experiment, "aaa");
+
+  const source = "test";
+  const slugs = [],
+    experiments = [];
+  for (let i = 0; i < 7; i++) {
+    slugs.push(`slug-${i}`);
+    experiments.push({
+      slug: slugs[i],
+      source,
+      branch: {
+        slug: "control",
+      },
+    });
+  }
+
+  manager.sessions.set(source, new Set([slugs[0]]));
+
+  manager._checkUnseenEnrollments(
+    experiments,
+    source,
+    [slugs[1]],
+    [slugs[2]],
+    new Map([]),
+    new Map([[slugs[3], experiments[3]]]),
+    [slugs[4]],
+    new Map([[slugs[5], experiments[5]]])
+  );
+
+  const events = Glean.nimbusEvents.enrollmentStatus.testGetValue();
+
+  Assert.equal(events?.length, 7);
+
+  Assert.equal(events[0].extra.status, "Enrolled");
+  Assert.equal(events[0].extra.reason, "Qualified");
+  Assert.equal(events[0].extra.branch, "control");
+  Assert.equal(events[0].extra.slug, slugs[0]);
+
+  Assert.equal(events[1].extra.status, "Disqualified");
+  Assert.equal(events[1].extra.reason, "NotTargeted");
+  Assert.equal(events[1].extra.branch, "control");
+  Assert.equal(events[1].extra.slug, slugs[1]);
+
+  Assert.equal(events[2].extra.status, "Disqualified");
+  Assert.equal(events[2].extra.reason, "Error");
+  Assert.equal(events[2].extra.error_string, "invalid-recipe");
+  Assert.equal(events[2].extra.branch, "control");
+  Assert.equal(events[2].extra.slug, slugs[2]);
+
+  Assert.equal(events[3].extra.status, "Disqualified");
+  Assert.equal(events[3].extra.reason, "Error");
+  Assert.equal(events[3].extra.error_string, "invalid-branch");
+  Assert.equal(events[3].extra.branch, "control");
+  Assert.equal(events[3].extra.slug, slugs[3]);
+
+  Assert.equal(events[4].extra.status, "Disqualified");
+  Assert.equal(events[4].extra.reason, "Error");
+  Assert.equal(events[4].extra.error_string, "l10n-missing-locale");
+  Assert.equal(events[4].extra.branch, "control");
+  Assert.equal(events[4].extra.slug, slugs[4]);
+
+  Assert.equal(events[5].extra.status, "Disqualified");
+  Assert.equal(events[5].extra.reason, "Error");
+  Assert.equal(events[5].extra.error_string, "l10n-missing-entry");
+  Assert.equal(events[5].extra.branch, "control");
+  Assert.equal(events[5].extra.slug, slugs[5]);
+
+  Assert.equal(events[6].extra.status, "WasEnrolled");
+  Assert.equal(events[6].extra.branch, "control");
+  Assert.equal(events[6].extra.slug, slugs[6]);
+
+  sandbox.restore();
+});