Bug 1721647: Add a stub page data service API to enable parallel development. r=mak

This adds the stub API for the page data service and some basic docs. The service can be used from
xpcshell tests and the events respond with sane data. As there are no consumers currently the
in-memory cache never clears.

Differential Revision: https://phabricator.services.mozilla.com/D120498

browser/base/content/test/static/browser_all_files_referenced.js

@@ -270,6 +270,9 @@ var whitelist = [
   },
   { file: "chrome://browser/content/screenshots/menu-fullpage.svg" },
   { file: "chrome://browser/content/screenshots/menu-visible.svg" },
+
+  // Will be resolved by bug 1722275.
+  { file: "resource://app/modules/pagedata/PageDataService.jsm" },
 ];
 
 if (AppConstants.NIGHTLY_BUILD && AppConstants.platform != "win") {

browser/components/moz.build

@@ -36,6 +36,7 @@ DIRS += [
     "downloads",
     "enterprisepolicies",
     "extensions",
+    "pagedata",
     "migration",
     "newtab",
     "originattributes",

browser/components/pagedata/.eslintrc.js

@@ -0,0 +1,42 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+module.exports = {
+  rules: {
+    "mozilla/var-only-at-top-level": "error",
+    "require-jsdoc": [
+      "error",
+      {
+        require: {
+          FunctionDeclaration: true,
+          MethodDefinition: true,
+          ClassDeclaration: true,
+          ArrowFunctionExpression: false,
+          FunctionExpression: false,
+        },
+      },
+    ],
+    "valid-jsdoc": [
+      "error",
+      {
+        prefer: {
+          return: "returns",
+        },
+        preferType: {
+          Boolean: "boolean",
+          Number: "number",
+          String: "string",
+          Object: "object",
+          bool: "boolean",
+        },
+        requireParamDescription: true,
+        requireReturn: false,
+        requireReturnDescription: false,
+      },
+    ],
+    "no-unused-expressions": "error",
+  },
+};

browser/components/pagedata/PageDataService.jsm

@@ -0,0 +1,136 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var EXPORTED_SYMBOLS = ["PageDataService"];
+
+const { XPCOMUtils } = ChromeUtils.import(
+  "resource://gre/modules/XPCOMUtils.jsm"
+);
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+  Services: "resource://gre/modules/Services.jsm",
+  EventEmitter: "resource://gre/modules/EventEmitter.jsm",
+});
+
+XPCOMUtils.defineLazyGetter(this, "logConsole", function() {
+  return console.createInstance({
+    prefix: "PageData",
+    maxLogLevel: Services.prefs.getBoolPref("browser.pagedata.log", false)
+      ? "Debug"
+      : "Warn",
+  });
+});
+
+/**
+ * @typedef {object} Data
+ *   An individual piece of data about a page.
+ * @property {string} type
+ *   The type of data.
+ * @property {object} data
+ *   The data in a format specific to the type of data.
+ *
+ * @typedef {object} PageData
+ *   A set of discovered from a page.
+ * @property {string} url
+ *   The page's url.
+ * @property {number} date
+ *   The epoch based timestamp for when the data was discovered.
+ * @property {Data[]} data
+ *   The array of data found which may be empty if no data was found.
+ */
+
+const PageDataService = new (class PageDataService extends EventEmitter {
+  /**
+   * Caches page data discovered from browsers. The key is the url of the data. Currently the cache
+   * never expires.
+   * @type {Map<string, PageData[]>}
+   */
+  #pageDataCache = new Map();
+
+  /**
+   * Constructs a new instance of the service, not called externally.
+   */
+  constructor() {
+    super();
+
+    if (!Services.prefs.getBoolPref("browser.pagedata.enabled", false)) {
+      return;
+    }
+
+    logConsole.debug("Service started");
+  }
+
+  /**
+   * Adds data for a url. This should generally only be called by other components of the
+   * page data service or tests for simulating page data collection.
+   *
+   * @param {string} url
+   *   The url of the page.
+   * @param {Data[]} data
+   *   The set of data discovered.
+   */
+  pageDataDiscovered(url, data) {
+    let pageData = {
+      url,
+      date: Date.now(),
+      data,
+    };
+
+    this.#pageDataCache.set(url, pageData);
+
+    // Send out a notification if there was some data found.
+    if (data.length) {
+      this.emit("page-data", pageData);
+    }
+  }
+
+  /**
+   * Retrieves any cached page data. Returns null if there is no information in the cache, this will
+   * happen either if the page has not been browsed recently or if data collection failed for some
+   * reason.
+   *
+   * @param {string} url
+   *   The url to retrieve data for.
+   * @returns {PageData|null}
+   *   A `PageData` if one is cached (it may not actually contain any items of data) or null if this
+   *   page has not been successfully checked for data recently.
+   */
+  getCached(url) {
+    return this.#pageDataCache.get(url) ?? null;
+  }
+
+  /**
+   * Queues page data retrieval for a url.
+   *
+   * @param {string} url
+   *   The url to retrieve data for.
+   * @returns {Promise<PageData>}
+   *   Resolves to a `PageData` (which may not contain any items of data) when the page has been
+   *   successfully checked for data. Will resolve immediately if there is cached data available.
+   *   Rejects if there was some failure to collect data.
+   */
+  async queueFetch(url) {
+    let cached = this.#pageDataCache.get(url);
+    if (cached) {
+      return cached;
+    }
+
+    let pageData = {
+      url,
+      date: Date.now(),
+      data: [],
+    };
+
+    this.#pageDataCache.set(url, pageData);
+
+    // Send out a notification if there was some data found.
+    if (pageData.data.length) {
+      this.emit("page-data", pageData);
+    }
+
+    return pageData;
+  }
+})();

browser/components/pagedata/docs/index.md

@@ -0,0 +1,42 @@
+# PageDataService
+
+The page data service is responsible for collecting additional data about a page. This could include
+information about the media on a page, product information, etc. When enabled it will automatically
+try to find page data for pages that the user browses or it can be directed to asynchronously look
+up the page data for a url.
+
+The `PageDataService` is an EventEmitter and listeners can subscribe to its notifications via the
+`on` and `once` methods.
+
+The service can be enabled by setting `browser.pagedata.enabled` to true. Additional logging can be
+enabled by setting `browser.pagedata.log` to true.
+
+## PageData Data Structure
+
+At a high level the page data service can collect many different kinds of data. When queried the
+service will respond with a `PageData` structure which holds the page's url the time when the data
+was discovered and an array of the different types of data found. This array will be empty if no
+data was found. Each item in the array is an object with two properties. `type` is a string
+indicating the type of data. `data` is the actual data, the specific format will vary depending on
+the type.
+
+## PageData Collection
+
+Page data is gathered in one of two ways.
+
+Page data is automatically gathered for webpages the user visits. This collection is trigged after
+a short delay and then updated when necessary. Any data is cached in memory for a period of time.
+When page data has been found a `page-data` event is emitted. The event's argument holds the
+`PageData` structure. The `getCached` function can be used to access any cached data for a url.
+
+Page data can also be requested for a URL that is not currently open. In this case the service will
+load the page in the background to find its data. The service operates a queueing system to reduce
+resource usage. As above when any new data is found the `page-data` event is emitted. The
+`queueFetch` method starts this process and returns a promise that resolves to the `PageData` or
+rejects in the event of failure.
+
+## Supported Types of page data
+
+The following types of page data are currently supported:
+
+... TBD

browser/components/pagedata/moz.build

@@ -0,0 +1,15 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+XPCSHELL_TESTS_MANIFESTS += [
+    "tests/unit/xpcshell.ini",
+]
+
+EXTRA_JS_MODULES.pagedata += [
+    "PageDataService.jsm",
+]
+
+SPHINX_TREES["docs"] = "docs"

browser/components/pagedata/tests/unit/test_pagedata_basic.js

@@ -0,0 +1,61 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * Simply tests that the notification is dispatched when new page data is
+ * discovered.
+ */
+
+const { XPCOMUtils } = ChromeUtils.import(
+  "resource://gre/modules/XPCOMUtils.jsm"
+);
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+  Services: "resource://gre/modules/Services.jsm",
+  PageDataService: "resource:///modules/pagedata/PageDataService.jsm",
+});
+
+add_task(async function notifies() {
+  let url = "https://www.mozilla.org/";
+
+  Assert.equal(
+    PageDataService.getCached(url),
+    null,
+    "Should be no cached data."
+  );
+
+  let listener = () => {
+    Assert.ok(false, "Should not notify for no data.");
+  };
+
+  PageDataService.on("page-data", listener);
+
+  let pageData = await PageDataService.queueFetch(url);
+  Assert.equal(pageData.url, "https://www.mozilla.org/");
+  Assert.equal(pageData.data.length, 0);
+
+  pageData = PageDataService.getCached(url);
+  Assert.equal(pageData.url, "https://www.mozilla.org/");
+  Assert.equal(pageData.data.length, 0);
+
+  PageDataService.off("page-data", listener);
+
+  let promise = PageDataService.once("page-data");
+
+  PageDataService.pageDataDiscovered(url, [
+    {
+      type: "product",
+      data: {
+        price: 276,
+      },
+    },
+  ]);
+
+  pageData = await promise;
+  Assert.equal(pageData.url, "https://www.mozilla.org/");
+  Assert.equal(pageData.data.length, 1);
+  Assert.equal(pageData.data[0].type, "product");
+
+  Assert.equal(PageDataService.getCached(url), pageData);
+  Assert.equal(await PageDataService.queueFetch(url), pageData);
+});

browser/components/pagedata/tests/unit/xpcshell.ini

@@ -0,0 +1,5 @@
+[DEFAULT]
+firefox-appdir = browser
+skip-if = toolkit == 'android'
+
+[test_pagedata_basic.js]

browser/docs/index.rst

@@ -14,6 +14,7 @@ This is the nascent documentation of the Firefox front-end code.
    components/newtab/docs/index
    installer/windows/installer/index
    /toolkit/mozapps/defaultagent/default-browser-agent/index
+   components/pagedata/docs/index
    components/newtab/content-src/asrouter/docs/index
    search/index
    base/sslerrorreport/index