Bug 1956901 - Generate typelib for all Glean metrics and pings r=Standard8,fkilic

Differential Revision: https://phabricator.services.mozilla.com/D243599

toolkit/components/nimbus/metrics.yaml

@@ -236,11 +236,6 @@ nimbus_targeting_context:
     notification_emails: *targeting_context_notification_emails
     send_in_pings: *targeting_context_pings
     description: The slugs of the actively enrolled rollouts
-    bugs: *targeting_context_bugs
-    data_reviews: *targeting_context_data_reviews
-    expires: *targeting_context_expiry
-    notification_emails: *targeting_context_notification_emails
-    send_in_pings: *targeting_context_pings
     data_sensitivity:
       - interaction
     type: object
@@ -270,11 +265,6 @@ nimbus_targeting_context:
     notification_emails: *targeting_context_notification_emails
     send_in_pings: *targeting_context_pings
     description: The architecture of the user's CPU (32-bit or 64-bit)
-    bugs: *targeting_context_bugs
-    data_reviews: *targeting_context_data_reviews
-    expires: *targeting_context_expiry
-    notification_emails: *targeting_context_notification_emails
-    send_in_pings: *targeting_context_pings
     data_sensitivity:
       - technical
     type: quantity

toolkit/components/resistfingerprinting/metrics.yaml

@@ -1560,42 +1560,6 @@ characteristics:
     data_sensitivity:
       - interaction
 
-  font_name_list_cursive_modified:
-    type: quantity
-    unit: int
-    description: >
-      The number of modified font.name-list.cursive. prefs.
-    lifetime: application
-    send_in_pings:
-      - user-characteristics
-    notification_emails:
-      - tom@mozilla.com
-    bugs:
-      - https://bugzilla.mozilla.org/show_bug.cgi?id=1887658
-    data_reviews:
-      - https://bugzilla.mozilla.org/show_bug.cgi?id=1887658#c4
-    expires: never
-    data_sensitivity:
-      - interaction
-
-  font_name_list_emoji_modified:
-    type: quantity
-    unit: int
-    description: >
-      The number of modified font.name-list.cursive. prefs.
-    lifetime: application
-    send_in_pings:
-      - user-characteristics
-    notification_emails:
-      - tom@mozilla.com
-    bugs:
-      - https://bugzilla.mozilla.org/show_bug.cgi?id=1887658
-    data_reviews:
-      - https://bugzilla.mozilla.org/show_bug.cgi?id=1887658#c4
-    expires: never
-    data_sensitivity:
-      - interaction
-
   font_name_list_emoji_modified:
     type: boolean
     description: >

tools/@types/glean/browser_components_search.d.ts

@@ -1,121 +0,0 @@
-/**
- * NOTE: Do not modify this file by hand.
- * Content was generated from source metrics.yaml files.
- * If you're updating some of the sources, see README for instructions.
- */
-
-interface GleanImpl {
-
-  newtabSearch: {
-    issued: GleanEvent;
-  }
-
-  newtabSearchAd: {
-    impression: GleanEvent;
-    click: GleanEvent;
-  }
-
-  sap: {
-    counts: GleanEvent;
-    deprecatedCounts: Record<string, GleanCounter>;
-  }
-
-  serp: {
-    impression: GleanEvent;
-    engagement: GleanEvent;
-    adImpression: GleanEvent;
-    abandonment: GleanEvent;
-    categorizationDuration: GleanTimingDistribution;
-    categorization: GleanEvent;
-    adsBlockedCount: Record<string, GleanCounter>;
-    experimentInfo: GleanObject;
-    categorizationNoMapFound: GleanCounter;
-  }
-
-  searchWith: {
-    reportingUrl: GleanUrl;
-    contextId: GleanUuid;
-  }
-
-  browserEngagementNavigation: {
-    urlbar: Record<string, GleanCounter>;
-    urlbarHandoff: Record<string, GleanCounter>;
-    urlbarPersisted: Record<string, GleanCounter>;
-    urlbarSearchmode: Record<string, GleanCounter>;
-    searchbar: Record<string, GleanCounter>;
-    aboutHome: Record<string, GleanCounter>;
-    aboutNewtab: Record<string, GleanCounter>;
-    contextmenu: Record<string, GleanCounter>;
-    webextension: Record<string, GleanCounter>;
-  }
-
-  browserSearchContent: {
-    urlbar: Record<string, GleanCounter>;
-    urlbarHandoff: Record<string, GleanCounter>;
-    urlbarPersisted: Record<string, GleanCounter>;
-    urlbarSearchmode: Record<string, GleanCounter>;
-    searchbar: Record<string, GleanCounter>;
-    aboutHome: Record<string, GleanCounter>;
-    aboutNewtab: Record<string, GleanCounter>;
-    contextmenu: Record<string, GleanCounter>;
-    webextension: Record<string, GleanCounter>;
-    system: Record<string, GleanCounter>;
-    tabhistory: Record<string, GleanCounter>;
-    reload: Record<string, GleanCounter>;
-    unknown: Record<string, GleanCounter>;
-  }
-
-  browserSearchWithads: {
-    urlbar: Record<string, GleanCounter>;
-    urlbarHandoff: Record<string, GleanCounter>;
-    urlbarPersisted: Record<string, GleanCounter>;
-    urlbarSearchmode: Record<string, GleanCounter>;
-    searchbar: Record<string, GleanCounter>;
-    aboutHome: Record<string, GleanCounter>;
-    aboutNewtab: Record<string, GleanCounter>;
-    contextmenu: Record<string, GleanCounter>;
-    webextension: Record<string, GleanCounter>;
-    system: Record<string, GleanCounter>;
-    tabhistory: Record<string, GleanCounter>;
-    reload: Record<string, GleanCounter>;
-    unknown: Record<string, GleanCounter>;
-  }
-
-  browserSearchAdclicks: {
-    urlbar: Record<string, GleanCounter>;
-    urlbarHandoff: Record<string, GleanCounter>;
-    urlbarPersisted: Record<string, GleanCounter>;
-    urlbarSearchmode: Record<string, GleanCounter>;
-    searchbar: Record<string, GleanCounter>;
-    aboutHome: Record<string, GleanCounter>;
-    aboutNewtab: Record<string, GleanCounter>;
-    contextmenu: Record<string, GleanCounter>;
-    webextension: Record<string, GleanCounter>;
-    system: Record<string, GleanCounter>;
-    tabhistory: Record<string, GleanCounter>;
-    reload: Record<string, GleanCounter>;
-    unknown: Record<string, GleanCounter>;
-  }
-
-  urlbarSearchmode: {
-    bookmarkmenu: Record<string, GleanCounter>;
-    handoff: Record<string, GleanCounter>;
-    keywordoffer: Record<string, GleanCounter>;
-    oneoff: Record<string, GleanCounter>;
-    searchbutton: Record<string, GleanCounter>;
-    shortcut: Record<string, GleanCounter>;
-    tabmenu: Record<string, GleanCounter>;
-    tabtosearch: Record<string, GleanCounter>;
-    tabtosearchOnboard: Record<string, GleanCounter>;
-    topsitesNewtab: Record<string, GleanCounter>;
-    topsitesUrlbar: Record<string, GleanCounter>;
-    touchbar: Record<string, GleanCounter>;
-    typed: Record<string, GleanCounter>;
-    historymenu: Record<string, GleanCounter>;
-    other: Record<string, GleanCounter>;
-  }
-
-  searchbar: {
-    selectedResultMethod: Record<string, GleanCounter>;
-  }
-}

tools/@types/glean/browser_components_search_pings.d.ts

@@ -1,10 +0,0 @@
-/**
- * NOTE: Do not modify this file by hand.
- * Content was generated from source metrics.yaml files.
- * If you're updating some of the sources, see README for instructions.
- */
-
-interface GleanPingsImpl {
-  searchWith: nsIGleanPingNoReason;
-  serpCategorization: nsIGleanPingWithReason<"startup"|"inactivity"|"threshold_reached">;
-}

tools/@types/glean/toolkit_components_extensions.d.ts

@@ -1,59 +0,0 @@
-/**
- * NOTE: Do not modify this file by hand.
- * Content was generated from source metrics.yaml files.
- * If you're updating some of the sources, see README for instructions.
- */
-
-interface GleanImpl {
-
-  extensions: {
-    useRemotePref: GleanBoolean;
-    useRemotePolicy: GleanBoolean;
-    startupCacheLoadTime: GleanTimespan;
-    startupCacheReadErrors: Record<string, GleanCounter>;
-    startupCacheWriteBytelength: GleanQuantity;
-    processEvent: Record<string, GleanCounter>;
-  }
-
-  extensionsApisDnr: {
-    startupCacheReadSize: GleanMemoryDistribution;
-    startupCacheReadTime: GleanTimingDistribution;
-    startupCacheWriteSize: GleanMemoryDistribution;
-    startupCacheWriteTime: GleanTimingDistribution;
-    startupCacheEntries: Record<string, GleanCounter>;
-    validateRulesTime: GleanTimingDistribution;
-    evaluateRulesTime: GleanTimingDistribution;
-    evaluateRulesCountMax: GleanQuantity;
-  }
-
-  extensionsData: {
-    migrateResult: GleanEvent;
-    storageLocalError: GleanEvent;
-    syncUsageQuotas: GleanEvent;
-    migrateResultCount: Record<string, GleanCounter>;
-  }
-
-  extensionsCounters: {
-    browserActionPreloadResult: Record<string, GleanCounter>;
-    eventPageIdleResult: Record<string, GleanCounter>;
-  }
-
-  extensionsTiming: {
-    backgroundPageLoad: GleanTimingDistribution;
-    backgroundPageLoadByAddonid: Record<string, GleanTimingDistribution>;
-    browserActionPopupOpen: GleanTimingDistribution;
-    browserActionPopupOpenByAddonid: Record<string, GleanTimingDistribution>;
-    contentScriptInjection: GleanTimingDistribution;
-    contentScriptInjectionByAddonid: Record<string, GleanTimingDistribution>;
-    eventPageRunningTime: GleanCustomDistribution;
-    eventPageRunningTimeByAddonid: Record<string, GleanTimingDistribution>;
-    extensionStartup: GleanTimingDistribution;
-    extensionStartupByAddonid: Record<string, GleanTimingDistribution>;
-    pageActionPopupOpen: GleanTimingDistribution;
-    pageActionPopupOpenByAddonid: Record<string, GleanTimingDistribution>;
-    storageLocalGetIdb: GleanTimingDistribution;
-    storageLocalGetIdbByAddonid: Record<string, GleanTimingDistribution>;
-    storageLocalSetIdb: GleanTimingDistribution;
-    storageLocalSetIdbByAddonid: Record<string, GleanTimingDistribution>;
-  }
-}

tools/@types/glean/toolkit_components_places.d.ts

@@ -1,39 +0,0 @@
-/**
- * NOTE: Do not modify this file by hand.
- * Content was generated from source metrics.yaml files.
- * If you're updating some of the sources, see README for instructions.
- */
-
-interface GleanImpl {
-
-  places: {
-    placesDatabaseCorruptionHandlingStage: Record<string, GleanString>;
-    sponsoredVisitNoTriggeringUrl: GleanCounter;
-    pagesNeedFrecencyRecalculation: GleanQuantity;
-    previousdayVisits: GleanQuantity;
-    pagesCount: GleanCustomDistribution;
-    mostRecentExpiredVisit: GleanTimingDistribution;
-    bookmarksCount: GleanCustomDistribution;
-    tagsCount: GleanCustomDistribution;
-    keywordsCount: GleanCustomDistribution;
-    backupsDaysfromlast: GleanTimingDistribution;
-    backupsBookmarkstree: GleanTimingDistribution;
-    backupsTojson: GleanTimingDistribution;
-    exportTohtml: GleanTimingDistribution;
-    sortedBookmarksPerc: GleanCustomDistribution;
-    taggedBookmarksPerc: GleanCustomDistribution;
-    databaseFilesize: GleanMemoryDistribution;
-    databaseFaviconsFilesize: GleanMemoryDistribution;
-    expirationStepsToClean: GleanCustomDistribution;
-    idleFrecencyDecayTime: GleanTimingDistribution;
-    idleMaintenanceTime: GleanTimingDistribution;
-    frecencyRecalcChunkTime: GleanTimingDistribution;
-    annosPagesCount: GleanCustomDistribution;
-    maintenanceDaysfromlast: GleanTimingDistribution;
-  }
-
-  pageIcon: {
-    smallIconCount: GleanCounter;
-    fitIconCount: GleanCounter;
-  }
-}

tools/@types/glean/toolkit_components_search.d.ts

@@ -1,47 +0,0 @@
-/**
- * NOTE: Do not modify this file by hand.
- * Content was generated from source metrics.yaml files.
- * If you're updating some of the sources, see README for instructions.
- */
-
-interface GleanImpl {
-
-  searchEngineDefault: {
-    engineId: GleanString;
-    displayName: GleanString;
-    loadPath: GleanString;
-    submissionUrl: GleanUrl;
-    changed: GleanEvent;
-  }
-
-  searchEnginePrivate: {
-    engineId: GleanString;
-    displayName: GleanString;
-    loadPath: GleanString;
-    submissionUrl: GleanUrl;
-    changed: GleanEvent;
-  }
-
-  searchService: {
-    startupTime: GleanTimingDistribution;
-    initializationStatus: Record<string, GleanCounter>;
-  }
-
-  browserSearchinit: {
-    engineInvalidWebextension: Record<string, GleanQuantity>;
-    secureOpensearchEngineCount: GleanQuantity;
-    insecureOpensearchEngineCount: GleanQuantity;
-    secureOpensearchUpdateCount: GleanQuantity;
-    insecureOpensearchUpdateCount: GleanQuantity;
-  }
-
-  search: {
-    suggestionsLatency: Record<string, GleanTimingDistribution>;
-  }
-
-  searchSuggestions: {
-    successfulRequests: Record<string, GleanCounter>;
-    abortedRequests: Record<string, GleanCounter>;
-    failedRequests: Record<string, GleanCounter>;
-  }
-}

tools/@types/glean/toolkit_mozapps_extensions.d.ts

@@ -1,27 +0,0 @@
-/**
- * NOTE: Do not modify this file by hand.
- * Content was generated from source metrics.yaml files.
- * If you're updating some of the sources, see README for instructions.
- */
-
-interface GleanImpl {
-
-  addonsManager: {
-    install: GleanEvent;
-    update: GleanEvent;
-    installStats: GleanEvent;
-    manage: GleanEvent;
-    reportSuspiciousSite: GleanEvent;
-  }
-
-  blocklist: {
-    lastModifiedRsAddonsMblf: GleanDatetime;
-    mlbfSource: GleanString;
-    mlbfSoftblocksSource: GleanString;
-    mlbfGenerationTime: GleanDatetime;
-    mlbfSoftblocksGenerationTime: GleanDatetime;
-    mlbfStashTimeOldest: GleanDatetime;
-    mlbfStashTimeNewest: GleanDatetime;
-    addonBlockChange: GleanEvent;
-  }
-}

tools/ts/build_glean.js

@@ -15,7 +15,7 @@ const YAML = require("yaml");
 
 const HEADER = `/**
  * NOTE: Do not modify this file by hand.
- * Content was generated from source metrics.yaml files.
+ * Content was generated from source glean .yaml files.
  * If you're updating some of the sources, see README for instructions.
  */
 `;
@@ -32,11 +32,13 @@ const TYPES = {
   timing_distribution: "GleanTimingDistribution",
   labeled_timing_distribution: "Record<string, GleanTimingDistribution>",
   memory_distribution: "GleanMemoryDistribution",
+  labeled_memory_distribution: "Record<string, GleanMemoryDistribution>",
   uuid: "GleanUuid",
   url: "GleanUrl",
   datetime: "GleanDatetime",
   event: "GleanEvent",
   custom_distribution: "GleanCustomDistribution",
+  labeled_custom_distribution: "Record<string, GleanCustomDistribution>",
   quantity: "GleanQuantity",
   labeled_quantity: "Record<string, GleanQuantity>",
   rate: "GleanRate",
@@ -78,8 +80,8 @@ function emitGlean(yamlDoc) {
 
 // Produce webidl types based on Glean pings.
 function emitGleanPings(yamlDoc) {
-  let lines = [HEADER];
-  lines.push("interface GleanPingsImpl {");
+  let lines = [];
+  lines.push("\ninterface GleanPingsImpl {");
 
   for (let [name, ping] of Object.entries(yamlDoc)) {
     if (!name.startsWith("$")) {
@@ -97,38 +99,33 @@ function emitGleanPings(yamlDoc) {
   return lines.join("\n");
 }
 
-// Build target .d.ts and update the Glean lib.
-async function main(src_dir, path, types_dir) {
-  if (!path.endsWith(".yaml")) {
-    path += "/metrics.yaml";
+// Build lib.gecko.glean.d.ts.
+async function main(src_dir, ...paths) {
+  let lib = `${src_dir}/tools/@types/lib.gecko.glean.d.ts`;
+
+  let metrics = {};
+  let pings = {};
+
+  for (let path of paths) {
+    console.log(`[INFO] ${path}`);
+    let yaml = fs.readFileSync(`${src_dir}/${path}`, "utf8");
+    let parsed = YAML.parse(yaml, { merge: true, schema: "failsafe" });
+
+    if (parsed.$schema === `${SCHEMA}/metrics/2-0-0`) {
+      for (let [key, val] of Object.entries(parsed)) {
+        if (typeof val === "object") {
+          Object.assign((metrics[key] ??= {}), val);
+        }
+      }
+    } else if (parsed.$schema === `${SCHEMA}/pings/2-0-0`) {
+      Object.assign(pings, parsed);
+    } else {
+      throw new Error(`Unknown Glean schema: ${parsed.$schema}`);
+    }
   }
 
-  let glean = `${src_dir}/${types_dir}/glean/`;
-  let lib = `${src_dir}/${types_dir}/lib.gecko.glean.d.ts`;
-  let dir = path.replaceAll("/", "_").replace(/_*([a-z]+.yaml)?$/, "");
-
-  let yaml = fs.readFileSync(`${src_dir}/${path}`, "utf8");
-  let parsed = YAML.parse(yaml, { merge: true, schema: "failsafe" });
-  let dts;
-
-  if (parsed.$schema === `${SCHEMA}/metrics/2-0-0`) {
-    dts = emitGlean(parsed);
-  } else if (parsed.$schema === `${SCHEMA}/pings/2-0-0`) {
-    dir += "_pings";
-    dts = emitGleanPings(parsed);
-  } else {
-    throw new Error(`Unknown Glean schema: ${parsed.$schema}`);
-  }
-
-  let target = `${glean}/${dir}.d.ts`;
-  console.log(`[INFO] ${target} (${dts.length.toLocaleString()} bytes)`);
-  fs.writeFileSync(target, dts);
-
-  let files = fs.readdirSync(glean).sort();
-  let refs = files.map(f => `/// <reference types="./glean/${f}" />`);
-  let all = `${HEADER}\n${refs.join("\n")}\n`;
-
-  console.log(`[INFO] ${lib} (${all.length.toLocaleString()} bytes)`);
+  let all = emitGlean(metrics) + emitGleanPings(pings);
+  console.log(`[WARN] ${lib} (${all.length.toLocaleString()} bytes)`);
   fs.writeFileSync(lib, all);
 }
 

tools/ts/mach_commands.py

@@ -120,10 +120,12 @@ def update(ctx):
 
 
 @SubCommand("ts", "glean", description="Build Glean bindings.")
-@CommandArgument("path", help="Path to a metrics.yaml or pings.yaml file.")
-def glean(ctx, path):
+def glean(ctx):
+    sys.path.append(mozpath.join(ctx.topsrcdir, "toolkit/components/glean/"))
+    from metrics_index import metrics_yamls, pings_yamls
+
     maybe_setup(ctx)
-    return node(ctx, "build_glean", ctx.topsrcdir, path, "tools/@types")
+    return node(ctx, "build_glean", ctx.topsrcdir, *metrics_yamls, *pings_yamls)
 
 
 @SubCommand("ts", "paths", description="Build module path mapping.")