Files
tubestation/browser/components/urlbar/UrlbarProviderQuickSuggest.sys.mjs
Drew Willcoxon 92fbc49d02 Bug 1942435 - Fix dismissals for Merino suggestions that aren't managed by a Suggest feature. r=daisuke
I don't think we have any tests specifically for Merino suggestions that don't
correspond to Suggest features, i.e., suggestions with a Merino provider that is
technically unrecognized by Firefox. Dynamic Wikipedia suggestions and
navigational suggestions (a.k.a. top sites) are two examples. So I added tasks
for both sponsored and nonsponsored suggestions to make sure they're disabled
when the relevant pref is disabled, in addition to testing this bug itself.

Differential Revision: https://phabricator.services.mozilla.com/D234787
2025-01-21 02:29:14 +00:00

622 lines
20 KiB
JavaScript

/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import {
UrlbarProvider,
UrlbarUtils,
} from "resource:///modules/UrlbarUtils.sys.mjs";
const lazy = {};
ChromeUtils.defineESModuleGetters(lazy, {
ContentRelevancyManager:
"resource://gre/modules/ContentRelevancyManager.sys.mjs",
QuickSuggest: "resource:///modules/QuickSuggest.sys.mjs",
SearchUtils: "resource://gre/modules/SearchUtils.sys.mjs",
UrlbarPrefs: "resource:///modules/UrlbarPrefs.sys.mjs",
UrlbarProviderSearchSuggestions:
"resource:///modules/UrlbarProviderSearchSuggestions.sys.mjs",
UrlbarResult: "resource:///modules/UrlbarResult.sys.mjs",
UrlbarSearchUtils: "resource:///modules/UrlbarSearchUtils.sys.mjs",
});
// Used for suggestions that don't otherwise have a score.
const DEFAULT_SUGGESTION_SCORE = 0.2;
/**
* A provider that returns a suggested url to the user based on what
* they have currently typed so they can navigate directly.
*/
class ProviderQuickSuggest extends UrlbarProvider {
/**
* Returns the name of this provider.
*
* @returns {string} the name of this provider.
*/
get name() {
return "UrlbarProviderQuickSuggest";
}
/**
* The type of the provider.
*
* @returns {UrlbarUtils.PROVIDER_TYPE}
*/
get type() {
return UrlbarUtils.PROVIDER_TYPE.NETWORK;
}
/**
* @returns {number}
* The default score for suggestions that don't otherwise have one. All
* suggestions require scores so they can be ranked. Scores are numeric
* values in the range [0, 1].
*/
get DEFAULT_SUGGESTION_SCORE() {
return DEFAULT_SUGGESTION_SCORE;
}
/**
* Whether this provider should be invoked for the given context.
* If this method returns false, the providers manager won't start a query
* with this provider, to save on resources.
*
* @param {UrlbarQueryContext} queryContext The query context object
* @returns {boolean} Whether this provider should be invoked for the search.
*/
isActive(queryContext) {
// If the sources don't include search or the user used a restriction
// character other than search, don't allow any suggestions.
if (
!queryContext.sources.includes(UrlbarUtils.RESULT_SOURCE.SEARCH) ||
(queryContext.restrictSource &&
queryContext.restrictSource != UrlbarUtils.RESULT_SOURCE.SEARCH)
) {
return false;
}
if (
!lazy.UrlbarPrefs.get("quickSuggestEnabled") ||
queryContext.isPrivate ||
queryContext.searchMode
) {
return false;
}
// Trim only the start of the search string because a trailing space can
// affect the suggestions.
let trimmedSearchString = queryContext.searchString.trimStart();
// Per product requirements, at least two characters must be typed to
// trigger a Suggest suggestion. Suggestion keywords should always be at
// least two characters long, but we check here anyway to be safe. Note we
// called `trimStart()` above, so we only call `trimEnd()` here.
if (trimmedSearchString.trimEnd().length < 2) {
return false;
}
this._trimmedSearchString = trimmedSearchString;
return true;
}
/**
* Starts querying. Extended classes should return a Promise resolved when the
* provider is done searching AND returning results.
*
* @param {UrlbarQueryContext} queryContext The query context object
* @param {Function} addCallback Callback invoked by the provider to add a new
* result. A UrlbarResult should be passed to it.
* @returns {Promise}
*/
async startQuery(queryContext, addCallback) {
let instance = this.queryInstance;
let searchString = this._trimmedSearchString;
// Fetch suggestions from all enabled backends.
let values = await Promise.all(
lazy.QuickSuggest.enabledBackends.map(backend =>
backend.query(searchString, { queryContext })
)
);
if (instance != this.queryInstance) {
return;
}
let suggestions = await this.#filterAndSortSuggestions(values.flat());
if (instance != this.queryInstance) {
return;
}
// Convert each suggestion into a result and add it. Don't add more than
// `maxResults` visible results so we don't spam the muxer.
let remainingCount = queryContext.maxResults ?? 10;
for (let suggestion of suggestions) {
if (!remainingCount) {
break;
}
let canAdd = await this.#canAddSuggestion(suggestion);
if (instance != this.queryInstance) {
return;
}
if (canAdd) {
let result = await this.#makeResult(queryContext, suggestion);
if (instance != this.queryInstance) {
return;
}
if (result) {
addCallback(this, result);
if (!result.isHiddenExposure) {
remainingCount--;
}
}
}
}
}
async #filterAndSortSuggestions(suggestions) {
let requiredKeys = ["source", "provider"];
let scoreMap = lazy.UrlbarPrefs.get("quickSuggestScoreMap");
let suggestionsByFeature = new Map();
let indexesBySuggestion = new Map();
for (let i = 0; i < suggestions.length; i++) {
let suggestion = suggestions[i];
// Discard suggestions that don't have the required keys, which are used
// to look up their features. Normally this shouldn't ever happen.
if (!requiredKeys.every(key => suggestion[key])) {
this.logger.error("Suggestion is missing one or more required keys", {
requiredKeys,
suggestion,
});
continue;
}
// Set `is_sponsored` before continuing because
// `#getSuggestionTelemetryType()` and other things depend on it.
let feature = this.#getFeature(suggestion);
if (!suggestion.hasOwnProperty("is_sponsored")) {
suggestion.is_sponsored = !!feature?.isSuggestionSponsored(suggestion);
}
// Ensure all suggestions have scores.
//
// Step 1: Set a default score if the suggestion doesn't have one.
if (typeof suggestion.score != "number" || isNaN(suggestion.score)) {
suggestion.score = DEFAULT_SUGGESTION_SCORE;
}
// Step 2: Apply relevancy ranking. For now we only do this for Merino
// suggestions, but we may expand it in the future.
if (suggestion.source == "merino") {
await this.#applyRanking(suggestion);
}
// Step 3: Apply score overrides defined in `quickSuggestScoreMap`. It
// maps telemetry types to scores.
if (scoreMap) {
let telemetryType = this.#getSuggestionTelemetryType(suggestion);
if (scoreMap.hasOwnProperty(telemetryType)) {
let score = parseFloat(scoreMap[telemetryType]);
if (!isNaN(score)) {
suggestion.score = score;
}
}
}
// Save some state used below to build the final list of suggestions.
let featureSuggestions = suggestionsByFeature.get(feature);
if (!featureSuggestions) {
featureSuggestions = [];
suggestionsByFeature.set(feature, featureSuggestions);
}
featureSuggestions.push(suggestion);
indexesBySuggestion.set(suggestion, i);
}
// Let each feature filter its suggestions.
suggestions = (
await Promise.all(
[...suggestionsByFeature].map(([feature, featureSuggestions]) =>
feature
? feature.filterSuggestions(featureSuggestions)
: Promise.resolve(featureSuggestions)
)
)
).flat();
// Sort the suggestions. When scores are equal, sort by original index to
// ensure a stable sort.
suggestions.sort((a, b) => {
return (
b.score - a.score ||
indexesBySuggestion.get(a) - indexesBySuggestion.get(b)
);
});
return suggestions;
}
onImpression(state, queryContext, controller, resultsAndIndexes, details) {
// Build a map from each feature to its results in `resultsAndIndexes`.
let resultsByFeature = resultsAndIndexes.reduce((memo, { result }) => {
let feature = this.#getFeatureByResult(result);
if (feature) {
let featureResults = memo.get(feature);
if (!featureResults) {
featureResults = [];
memo.set(feature, featureResults);
}
featureResults.push(result);
}
return memo;
}, new Map());
// Notify each feature with its results.
for (let [feature, featureResults] of resultsByFeature) {
feature.onImpression(
state,
queryContext,
controller,
featureResults,
details
);
}
}
onEngagement(queryContext, controller, details) {
let { result } = details;
// Delegate to the result's feature if there is one.
let feature = this.#getFeatureByResult(result);
if (feature) {
feature.onEngagement(
queryContext,
controller,
details,
this._trimmedSearchString
);
return;
}
// Otherwise, handle commands. The dismiss, manage, and help commands are
// supported for results without features. Dismissal is the only one we need
// to handle here since urlbar handles the others.
if (details.selType == "dismiss" && result.payload.isBlockable) {
lazy.QuickSuggest.blockedSuggestions.add(result.payload.url);
controller.removeResult(result);
}
}
onSearchSessionEnd(queryContext, controller, details) {
for (let backend of lazy.QuickSuggest.enabledBackends) {
backend.onSearchSessionEnd(queryContext, controller, details);
}
}
/**
* This is called only for dynamic result types, when the urlbar view updates
* the view of one of the results of the provider. It should return an object
* describing the view update.
*
* @param {UrlbarResult} result The result whose view will be updated.
* @returns {object} An object describing the view update.
*/
getViewUpdate(result) {
return this.#getFeatureByResult(result)?.getViewUpdate?.(result);
}
getResultCommands(result) {
return this.#getFeatureByResult(result)?.getResultCommands?.(result);
}
/**
* Gets the `BaseFeature` instance that implements suggestions for a source
* and provider name. The source and provider name can be supplied from either
* a suggestion object or the payload of a `UrlbarResult` object.
*
* @param {object} options
* Options object.
* @param {string} options.source
* The suggestion source, one of: "merino", "ml", "rust"
* @param {string} options.provider
* This value depends on `source`. The possible values per source are:
*
* merino:
* The name of the Merino provider that serves the suggestion type
* ml:
* The name of the intent as determined by `MLSuggest`
* rust:
* The name of the suggestion type as defined in Rust
* @returns {BaseFeature}
* The feature instance or null if no feature was found.
*/
#getFeature({ source, provider }) {
switch (source) {
case "merino":
return lazy.QuickSuggest.getFeatureByMerinoProvider(provider);
case "rust":
return lazy.QuickSuggest.getFeatureByRustSuggestionType(provider);
case "ml":
return lazy.QuickSuggest.getFeatureByMlIntent(provider);
}
return null;
}
#getFeatureByResult(result) {
return this.#getFeature(result.payload);
}
/**
* Returns the telemetry type for a suggestion. A telemetry type uniquely
* identifies a type of suggestion as well as the kind of `UrlbarResult`
* instances created from it.
*
* @param {object} suggestion
* A suggestion from a Suggest backend.
* @returns {string}
* The telemetry type. If the suggestion type is managed by a `BaseFeature`
* instance, the telemetry type is retrieved from it. Otherwise the
* suggestion type is assumed to come from Merino, and `suggestion.provider`
* (the Merino provider name) is returned.
*/
#getSuggestionTelemetryType(suggestion) {
let feature = this.#getFeature(suggestion);
if (feature) {
return feature.getSuggestionTelemetryType(suggestion);
}
return suggestion.provider;
}
async #makeResult(queryContext, suggestion) {
let result;
let feature = this.#getFeature(suggestion);
if (!feature) {
// We specifically allow Merino to serve suggestion types that Firefox
// doesn't know about so that we can experiment with new types without
// requiring changes in Firefox. No other source should return unknown
// suggestion types with the possible exception of the ML backend: Its
// models are stored in remote settings and it may return newer intents
// that aren't recognized by older Firefoxes.
if (suggestion.source != "merino") {
return null;
}
result = this.#makeDefaultResult(queryContext, suggestion);
} else {
result = await feature.makeResult(
queryContext,
suggestion,
this._trimmedSearchString
);
if (!result) {
// Feature might return null, if the feature is disabled and so on.
return null;
}
}
// See `#getFeature()` for possible values of `source` and `provider`.
result.payload.source = suggestion.source;
result.payload.provider = suggestion.provider;
result.payload.telemetryType = this.#getSuggestionTelemetryType(suggestion);
// Handle icons here so each feature doesn't have to do it, but use `||=` to
// let them do it if they need to.
result.payload.icon ||= suggestion.icon;
result.payload.iconBlob ||= suggestion.icon_blob;
// Set the appropriate suggested index and related properties unless the
// feature did it already.
if (!result.hasSuggestedIndex) {
if (suggestion.is_top_pick) {
result.isBestMatch = true;
result.isRichSuggestion = true;
result.richSuggestionIconSize ||= 52;
result.suggestedIndex = 1;
} else if (
!isNaN(suggestion.position) &&
lazy.UrlbarPrefs.get("quickSuggestAllowPositionInSuggestions")
) {
result.suggestedIndex = suggestion.position;
} else {
result.isSuggestedIndexRelativeToGroup = true;
if (!suggestion.is_sponsored) {
result.suggestedIndex = lazy.UrlbarPrefs.get(
"quickSuggestNonSponsoredIndex"
);
} else if (
lazy.UrlbarPrefs.get("showSearchSuggestionsFirst") &&
lazy.UrlbarProviderSearchSuggestions.isActive(queryContext) &&
lazy.UrlbarSearchUtils.getDefaultEngine(
queryContext.isPrivate
).supportsResponseType(lazy.SearchUtils.URL_TYPE.SUGGEST_JSON)
) {
// Show sponsored suggestions somewhere other than the bottom of the
// Suggest section only if search suggestions are shown first, the
// search suggestions provider is active for the current context (it
// will not be active if search suggestions are disabled, among other
// reasons), and the default engine supports suggestions.
result.suggestedIndex = lazy.UrlbarPrefs.get(
"quickSuggestSponsoredIndex"
);
} else {
result.suggestedIndex = -1;
}
}
}
return result;
}
#makeDefaultResult(queryContext, suggestion) {
let payload = {
url: suggestion.url,
isSponsored: suggestion.is_sponsored,
isBlockable: true,
blockL10n: {
id: "urlbar-result-menu-dismiss-firefox-suggest",
},
isManageable: true,
};
if (suggestion.full_keyword) {
payload.title = suggestion.title;
payload.qsSuggestion = [
suggestion.full_keyword,
UrlbarUtils.HIGHLIGHT.SUGGESTED,
];
} else {
payload.title = [suggestion.title, UrlbarUtils.HIGHLIGHT.TYPED];
payload.shouldShowUrl = true;
}
return new lazy.UrlbarResult(
UrlbarUtils.RESULT_TYPE.URL,
UrlbarUtils.RESULT_SOURCE.SEARCH,
...lazy.UrlbarResult.payloadAndSimpleHighlights(
queryContext.tokens,
payload
)
);
}
/**
* Cancels the current query.
*/
cancelQuery() {
for (let backend of lazy.QuickSuggest.enabledBackends) {
backend.cancelQuery();
}
}
/**
* Applies relevancy ranking to a suggestion by updating its score.
*
* @param {object} suggestion
* The suggestion to be ranked.
*/
async #applyRanking(suggestion) {
let oldScore = suggestion.score;
let mode = lazy.UrlbarPrefs.get("quickSuggestRankingMode");
switch (mode) {
case "random":
suggestion.score = Math.random();
break;
case "interest":
await this.#updateScoreByRelevance(suggestion);
break;
case "default":
default:
// Do nothing.
return;
}
this.logger.debug("Applied ranking to suggestion score", {
mode,
oldScore,
newScore: suggestion.score.toFixed(3),
});
}
/**
* Update score by interest-based relevance scoring. The final score is a mean
* between the interest-based score and the default static score, which means
* if the former is 0 or less than the latter, the combined score will be less
* than the static score.
*
* @param {object} suggestion
* The suggestion to be ranked.
*/
async #updateScoreByRelevance(suggestion) {
if (!suggestion.categories?.length) {
return;
}
let score;
try {
score = await lazy.ContentRelevancyManager.score(
suggestion.categories,
true // adjustment needed b/c Merino uses the original encoding
);
} catch (error) {
Glean.suggestRelevance.status.failure.add(1);
this.logger.error("Error updating suggestion score", error);
return;
}
Glean.suggestRelevance.status.success.add(1);
let oldScore = suggestion.score;
suggestion.score = (oldScore + score) / 2;
Glean.suggestRelevance.outcome[
suggestion.score >= oldScore ? "boosted" : "decreased"
].add(1);
}
/**
* Returns whether a given suggestion can be added for a query, assuming the
* provider itself should be active.
*
* @param {object} suggestion
* The suggestion to check.
* @returns {boolean}
* Whether the suggestion can be added.
*/
async #canAddSuggestion(suggestion) {
this.logger.debug("Checking if suggestion can be added", suggestion);
// Return false if suggestions are disabled. Always allow Rust exposure
// suggestions.
if (
((suggestion.is_sponsored &&
!lazy.UrlbarPrefs.get("suggest.quicksuggest.sponsored")) ||
(!suggestion.is_sponsored &&
!lazy.UrlbarPrefs.get("suggest.quicksuggest.nonsponsored"))) &&
(suggestion.source != "rust" || suggestion.provider != "Exposure")
) {
this.logger.debug("Suggestions disabled, not adding suggestion");
return false;
}
// Return false if an impression cap has been hit.
if (
(suggestion.is_sponsored &&
lazy.UrlbarPrefs.get("quickSuggestImpressionCapsSponsoredEnabled")) ||
(!suggestion.is_sponsored &&
lazy.UrlbarPrefs.get("quickSuggestImpressionCapsNonSponsoredEnabled"))
) {
let type = suggestion.is_sponsored ? "sponsored" : "nonsponsored";
let hitStats = lazy.QuickSuggest.impressionCaps.getHitStats(type);
if (hitStats) {
this.logger.debug("Impression cap(s) hit, not adding suggestion", {
type,
hitStats,
});
return false;
}
}
// Return false if the suggestion is blocked based on its URL. Suggestions
// from the JS backend define a single `url` property. Suggestions from the
// Rust backend are more complicated: Sponsored suggestions define `rawUrl`,
// which may contain timestamp templates, while non-sponsored suggestions
// define only `url`. Blocking should always be based on URLs with timestamp
// templates, where applicable, so check `rawUrl` and then `url`, in that
// order.
let { blockedSuggestions } = lazy.QuickSuggest;
if (await blockedSuggestions.has(suggestion.rawUrl ?? suggestion.url)) {
this.logger.debug("Suggestion blocked, not adding suggestion");
return false;
}
this.logger.debug("Suggestion can be added");
return true;
}
async _test_applyRanking(suggestion) {
await this.#applyRanking(suggestion);
}
}
export var UrlbarProviderQuickSuggest = new ProviderQuickSuggest();