4259d07cde
The Suggest Rust component currently stores the SQLite database in the
"local profile" directory, since it doesn't contain any user data.
Now that the component supports remembering dismissed suggestions,
we'll want to store the database in the "profile" directory instead,
since it'll contain persistent user data.
This commit prepares us for that future by specifying the profile
directory as the data path when we build the store. This commit also
adopts the `SuggestStoreBuilder#remoteSettings{Server, BucketName}()`
APIs.
Depends on D201774
Differential Revision: https://phabricator.services.mozilla.com/D203824
427 lines
14 KiB
JavaScript
427 lines
14 KiB
JavaScript
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
|
|
import { BaseFeature } from "resource:///modules/urlbar/private/BaseFeature.sys.mjs";
|
|
import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
|
|
|
|
const lazy = {};
|
|
|
|
ChromeUtils.defineESModuleGetters(lazy, {
|
|
QuickSuggest: "resource:///modules/QuickSuggest.sys.mjs",
|
|
RemoteSettingsServer: "resource://gre/modules/RustSuggest.sys.mjs",
|
|
SuggestIngestionConstraints: "resource://gre/modules/RustSuggest.sys.mjs",
|
|
SuggestStoreBuilder: "resource://gre/modules/RustSuggest.sys.mjs",
|
|
Suggestion: "resource://gre/modules/RustSuggest.sys.mjs",
|
|
SuggestionProvider: "resource://gre/modules/RustSuggest.sys.mjs",
|
|
SuggestionQuery: "resource://gre/modules/RustSuggest.sys.mjs",
|
|
UrlbarPrefs: "resource:///modules/UrlbarPrefs.sys.mjs",
|
|
Utils: "resource://services-settings/Utils.sys.mjs",
|
|
});
|
|
|
|
XPCOMUtils.defineLazyServiceGetter(
|
|
lazy,
|
|
"timerManager",
|
|
"@mozilla.org/updates/timer-manager;1",
|
|
"nsIUpdateTimerManager"
|
|
);
|
|
|
|
const SUGGEST_DATA_STORE_BASENAME = "suggest.sqlite";
|
|
|
|
// This ID is used to register our ingest timer with nsIUpdateTimerManager.
|
|
const INGEST_TIMER_ID = "suggest-ingest";
|
|
const INGEST_TIMER_LAST_UPDATE_PREF = `app.update.lastUpdateTime.${INGEST_TIMER_ID}`;
|
|
|
|
// Maps from `suggestion.constructor` to the corresponding name of the
|
|
// suggestion type. See `getSuggestionType()` for details.
|
|
const gSuggestionTypesByCtor = new WeakMap();
|
|
|
|
/**
|
|
* The Suggest Rust backend. Not used when the remote settings JS backend is
|
|
* enabled.
|
|
*
|
|
* This class returns suggestions served by the Rust component. These are the
|
|
* primary related architectural pieces (see bug 1851256 for details):
|
|
*
|
|
* (1) The `suggest` Rust component, which lives in the application-services
|
|
* repo [1] and is periodically vendored into mozilla-central [2] and then
|
|
* built into the Firefox binary.
|
|
* (2) `suggest.udl`, which is part of the Rust component's source files and
|
|
* defines the interface exposed to foreign-function callers like JS [3, 4].
|
|
* (3) `RustSuggest.sys.mjs` [5], which contains the JS bindings generated from
|
|
* `suggest.udl` by UniFFI. The classes defined in `RustSuggest.sys.mjs` are
|
|
* what we consume here in this file. If you have a question about the JS
|
|
* interface to the Rust component, try checking `RustSuggest.sys.mjs`, but
|
|
* as you get accustomed to UniFFI JS conventions you may find it simpler to
|
|
* refer directly to `suggest.udl`.
|
|
* (4) `config.toml` [6], which defines which functions in the JS bindings are
|
|
* sync and which are async. Functions default to the "worker" thread, which
|
|
* means they are async. Some functions are "main", which means they are
|
|
* sync. Async functions return promises. This information is reflected in
|
|
* `RustSuggest.sys.mjs` of course: If a function is "worker", its JS
|
|
* binding will return a promise, and if it's "main" it won't.
|
|
*
|
|
* [1] https://github.com/mozilla/application-services/tree/main/components/suggest
|
|
* [2] https://searchfox.org/mozilla-central/source/third_party/rust/suggest
|
|
* [3] https://github.com/mozilla/application-services/blob/main/components/suggest/src/suggest.udl
|
|
* [4] https://searchfox.org/mozilla-central/source/third_party/rust/suggest/src/suggest.udl
|
|
* [5] https://searchfox.org/mozilla-central/source/toolkit/components/uniffi-bindgen-gecko-js/components/generated/RustSuggest.sys.mjs
|
|
* [6] https://searchfox.org/mozilla-central/source/toolkit/components/uniffi-bindgen-gecko-js/config.toml
|
|
*/
|
|
export class SuggestBackendRust extends BaseFeature {
|
|
/**
|
|
* @returns {object}
|
|
* The global Suggest config from the Rust component as returned from
|
|
* `SuggestStore.fetchGlobalConfig()`.
|
|
*/
|
|
get config() {
|
|
return this.#config || {};
|
|
}
|
|
|
|
/**
|
|
* @returns {Promise}
|
|
* If ingest is pending this will be resolved when it's done. Otherwise it
|
|
* was resolved when the previous ingest finished.
|
|
*/
|
|
get ingestPromise() {
|
|
return this.#ingestPromise;
|
|
}
|
|
|
|
get shouldEnable() {
|
|
return lazy.UrlbarPrefs.get("quickSuggestRustEnabled");
|
|
}
|
|
|
|
enable(enabled) {
|
|
if (enabled) {
|
|
this.#init();
|
|
} else {
|
|
this.#uninit();
|
|
}
|
|
}
|
|
|
|
async query(searchString) {
|
|
this.logger.info("Handling query: " + JSON.stringify(searchString));
|
|
|
|
if (!this.#store) {
|
|
// There must have been an error creating `#store`.
|
|
this.logger.info("#store is null, returning");
|
|
return [];
|
|
}
|
|
|
|
// Build the list of enabled Rust providers to query.
|
|
let providers = this.#rustProviders.reduce(
|
|
(memo, { type, feature, provider }) => {
|
|
if (feature.isEnabled && feature.isRustSuggestionTypeEnabled(type)) {
|
|
this.logger.debug(
|
|
`Adding provider to query: '${type}' (${provider})`
|
|
);
|
|
memo.push(provider);
|
|
}
|
|
return memo;
|
|
},
|
|
[]
|
|
);
|
|
|
|
let suggestions = await this.#store.query(
|
|
new lazy.SuggestionQuery({ keyword: searchString, providers })
|
|
);
|
|
|
|
for (let suggestion of suggestions) {
|
|
let type = getSuggestionType(suggestion);
|
|
if (!type) {
|
|
continue;
|
|
}
|
|
|
|
suggestion.source = "rust";
|
|
suggestion.provider = type;
|
|
suggestion.is_sponsored = type == "Amp" || type == "Yelp";
|
|
if (Array.isArray(suggestion.icon)) {
|
|
suggestion.icon_blob = new Blob([new Uint8Array(suggestion.icon)], {
|
|
type: suggestion.iconMimetype ?? "",
|
|
});
|
|
|
|
delete suggestion.icon;
|
|
delete suggestion.iconMimetype;
|
|
}
|
|
}
|
|
|
|
this.logger.debug(
|
|
"Got suggestions: " + JSON.stringify(suggestions, null, 2)
|
|
);
|
|
|
|
return suggestions;
|
|
}
|
|
|
|
cancelQuery() {
|
|
this.#store?.interrupt();
|
|
}
|
|
|
|
/**
|
|
* Returns suggestion-type-specific configuration data set by the Rust
|
|
* backend.
|
|
*
|
|
* @param {string} type
|
|
* A Rust suggestion type name as defined in `suggest.udl`, e.g., "Amp",
|
|
* "Wikipedia", "Mdn", etc. See also `BaseFeature.rustSuggestionTypes`.
|
|
* @returns {object} config
|
|
* The config data for the type.
|
|
*/
|
|
getConfigForSuggestionType(type) {
|
|
return this.#configsBySuggestionType.get(type);
|
|
}
|
|
|
|
/**
|
|
* nsITimerCallback
|
|
*/
|
|
notify() {
|
|
this.logger.info("Ingest timer fired");
|
|
this.#ingest();
|
|
}
|
|
|
|
get #storeDataPath() {
|
|
return PathUtils.join(
|
|
Services.dirsvc.get("ProfD", Ci.nsIFile).path,
|
|
SUGGEST_DATA_STORE_BASENAME
|
|
);
|
|
}
|
|
|
|
/**
|
|
* @returns {Array}
|
|
* Each item in this array contains metadata related to a Rust suggestion
|
|
* type, the `BaseFeature` that manages the type, and the corresponding
|
|
* suggestion provider as defined by Rust. Items look like this:
|
|
* `{ type, feature, provider }`
|
|
*
|
|
* {string} type
|
|
* The Rust suggestion type name (the same type of string values that are
|
|
* defined in `BaseFeature.rustSuggestionTypes`).
|
|
* {BaseFeature} feature
|
|
* The feature that manages the suggestion type.
|
|
* {number} provider
|
|
* An integer value defined on the `SuggestionProvider` object in
|
|
* `RustSuggest.sys.mjs` that identifies the suggestion provider to
|
|
* Rust.
|
|
*/
|
|
get #rustProviders() {
|
|
let items = [];
|
|
for (let [type, feature] of lazy.QuickSuggest
|
|
.featuresByRustSuggestionType) {
|
|
let key = type.toUpperCase();
|
|
if (!lazy.SuggestionProvider.hasOwnProperty(key)) {
|
|
this.logger.error(`SuggestionProvider["${key}"] is not defined!`);
|
|
continue;
|
|
}
|
|
items.push({ type, feature, provider: lazy.SuggestionProvider[key] });
|
|
}
|
|
return items;
|
|
}
|
|
|
|
async #init() {
|
|
// Important note on schema updates:
|
|
//
|
|
// The first time the Suggest store is accessed after a schema version
|
|
// update, its backing database will be deleted and a new empty database
|
|
// will be created. The database will remain empty until we tell the store
|
|
// to ingest. If we wait to ingest as usual until our ingest timer fires,
|
|
// the store will remain empty for up to 24 hours, which means we won't
|
|
// serve any suggestions at all during that time.
|
|
//
|
|
// Therefore we simply always ingest here in `#init()`. We'll sometimes
|
|
// ingest unnecessarily but that's better than the alternative. (As a
|
|
// reminder, for users who have Suggest enabled `#init()` is called whenever
|
|
// the Rust backend is enabled, including on startup.)
|
|
|
|
// Initialize the store.
|
|
this.logger.info(
|
|
`Initializing SuggestStore with data path ${this.#storeDataPath}`
|
|
);
|
|
|
|
let builder = lazy.SuggestStoreBuilder.init()
|
|
.dataPath(this.#storeDataPath)
|
|
.remoteSettingsServer(this.#remoteSettingsServer)
|
|
.remoteSettingsBucketName(this.#remoteSettingsBucketName);
|
|
try {
|
|
this.#store = builder.build();
|
|
} catch (error) {
|
|
this.logger.error("Error initializing SuggestStore:");
|
|
this.logger.error(error);
|
|
return;
|
|
}
|
|
|
|
// Log the last ingest time for debugging.
|
|
let lastIngestSecs = Services.prefs.getIntPref(
|
|
INGEST_TIMER_LAST_UPDATE_PREF,
|
|
0
|
|
);
|
|
if (lastIngestSecs) {
|
|
this.logger.debug(
|
|
`Last ingest time: ${lastIngestSecs}s (${
|
|
Math.round(Date.now() / 1000) - lastIngestSecs
|
|
}s ago)`
|
|
);
|
|
} else {
|
|
this.logger.debug("Last ingest time: none");
|
|
}
|
|
|
|
// Register the ingest timer.
|
|
lazy.timerManager.registerTimer(
|
|
INGEST_TIMER_ID,
|
|
this,
|
|
lazy.UrlbarPrefs.get("quicksuggest.rustIngestIntervalSeconds"),
|
|
true // skipFirst
|
|
);
|
|
|
|
// Ingest.
|
|
await this.#ingest();
|
|
}
|
|
|
|
#uninit() {
|
|
this.#store = null;
|
|
this.#configsBySuggestionType.clear();
|
|
lazy.timerManager.unregisterTimer(INGEST_TIMER_ID);
|
|
}
|
|
|
|
async #ingest() {
|
|
let instance = (this.#ingestInstance = {});
|
|
await this.#ingestPromise;
|
|
if (instance != this.#ingestInstance) {
|
|
return;
|
|
}
|
|
this.#ingestPromise = new Promise(resolve => {
|
|
ChromeUtils.idleDispatch(() => this.#ingestHelper().finally(resolve));
|
|
});
|
|
await this.#ingestPromise;
|
|
}
|
|
|
|
async #ingestHelper() {
|
|
if (!this.#store) {
|
|
return;
|
|
}
|
|
|
|
this.logger.info("Starting ingest and configs fetch");
|
|
|
|
// Do the ingest.
|
|
this.logger.debug("Starting ingest");
|
|
try {
|
|
await this.#store.ingest(new lazy.SuggestIngestionConstraints());
|
|
} catch (error) {
|
|
// Ingest can throw a `SuggestApiError` subclass called `Other` that has a
|
|
// custom `reason` message, which is very helpful for diagnosing problems
|
|
// with remote settings data in tests in particular.
|
|
this.logger.error("Ingest error: " + (error.reason ?? error));
|
|
}
|
|
this.logger.debug("Finished ingest");
|
|
|
|
if (!this.#store) {
|
|
this.logger.info("#store became null, returning from ingest");
|
|
return;
|
|
}
|
|
|
|
// Fetch the global config.
|
|
this.logger.debug("Fetching global config");
|
|
this.#config = await this.#store.fetchGlobalConfig();
|
|
this.logger.debug("Got global config: " + JSON.stringify(this.#config));
|
|
|
|
if (!this.#store) {
|
|
this.logger.info("#store became null, returning from ingest");
|
|
return;
|
|
}
|
|
|
|
// Fetch all provider configs. We do this for all features, even ones that
|
|
// are currently disabled, because they may become enabled before the next
|
|
// ingest.
|
|
this.logger.debug("Fetching provider configs");
|
|
await Promise.all(
|
|
this.#rustProviders.map(async ({ type, provider }) => {
|
|
let config = await this.#store.fetchProviderConfig(provider);
|
|
this.logger.debug(
|
|
`Got '${type}' provider config: ` + JSON.stringify(config)
|
|
);
|
|
this.#configsBySuggestionType.set(type, config);
|
|
})
|
|
);
|
|
this.logger.debug("Finished fetching provider configs");
|
|
|
|
this.logger.info("Finished ingest and configs fetch");
|
|
}
|
|
|
|
async _test_setRemoteSettingsConfig({ serverUrl, bucketName }) {
|
|
this.#remoteSettingsServer = new lazy.RemoteSettingsServer.Custom(
|
|
serverUrl
|
|
);
|
|
this.#remoteSettingsBucketName = bucketName;
|
|
|
|
if (this.isEnabled) {
|
|
// Recreate the store and re-ingest.
|
|
Services.prefs.clearUserPref(INGEST_TIMER_LAST_UPDATE_PREF);
|
|
this.#uninit();
|
|
await this.#init();
|
|
}
|
|
}
|
|
|
|
async _test_ingest() {
|
|
await this.#ingest();
|
|
}
|
|
|
|
// The `SuggestStore` instance.
|
|
#store;
|
|
|
|
// Global Suggest config as returned from `SuggestStore.fetchGlobalConfig()`.
|
|
#config = {};
|
|
|
|
// Maps from suggestion type to provider config as returned from
|
|
// `SuggestStore.fetchProviderConfig()`.
|
|
#configsBySuggestionType = new Map();
|
|
|
|
#ingestPromise;
|
|
#ingestInstance;
|
|
#remoteSettingsServer = new lazy.RemoteSettingsServer.Custom(
|
|
lazy.Utils.SERVER_URL
|
|
);
|
|
#remoteSettingsBucketName = lazy.Utils.actualBucketName("main");
|
|
}
|
|
|
|
/**
|
|
* Returns the type of a suggestion.
|
|
*
|
|
* @param {Suggestion} suggestion
|
|
* A suggestion object, an instance of one of the `Suggestion` subclasses.
|
|
* @returns {string}
|
|
* The suggestion's type, e.g., "Amp", "Wikipedia", etc.
|
|
*/
|
|
function getSuggestionType(suggestion) {
|
|
// Suggestion objects served by the Rust component don't have any inherent
|
|
// type information other than the classes they are instances of. There's no
|
|
// `type` property, for example. There's a base `Suggestion` class and many
|
|
// `Suggestion` subclasses, one per type of suggestion. Each suggestion object
|
|
// is an instance of one of these subclasses. We derive a suggestion's type
|
|
// from the subclass it's an instance of.
|
|
//
|
|
// Unfortunately the subclasses are all anonymous, which means
|
|
// `suggestion.constructor.name` is always an empty string. (This is due to
|
|
// how UniFFI generates JS bindings.) Instead, the subclasses are defined as
|
|
// properties on the base `Suggestion` class. For example,
|
|
// `Suggestion.Wikipedia` is the (anonymous) Wikipedia suggestion class. To
|
|
// find a suggestion's subclass, we loop through the keys on `Suggestion`
|
|
// until we find the value the suggestion is an instance of. To avoid doing
|
|
// this every time, we cache the mapping from suggestion constructor to key
|
|
// the first time we encounter a new suggestion subclass.
|
|
let type = gSuggestionTypesByCtor.get(suggestion.constructor);
|
|
if (!type) {
|
|
type = Object.keys(lazy.Suggestion).find(
|
|
key => suggestion instanceof lazy.Suggestion[key]
|
|
);
|
|
if (type) {
|
|
gSuggestionTypesByCtor.set(suggestion.constructor, type);
|
|
} else {
|
|
this.logger.error(
|
|
"Unexpected error: Suggestion class not found on `Suggestion`. " +
|
|
"Did the Rust component or its JS bindings change? " +
|
|
"The suggestion is: " +
|
|
JSON.stringify(suggestion)
|
|
);
|
|
}
|
|
}
|
|
return type;
|
|
}
|