Bug 1965153 - Interest model implementation Interest model implementation for new tab content p13n r=thecount,home-newtab-reviewers
Differential Revision: https://phabricator.services.mozilla.com/D248465
This commit is contained in:
Rolf Rando
committed by
sdowne@mozilla.com
sdowne@mozilla.com
parent
0e60efc57e
commit
740b82a0cb
@@ -1974,6 +1974,8 @@ pref("browser.newtabpage.activity-stream.discoverystream.sections.personalizatio
|
||||
pref("browser.newtabpage.activity-stream.discoverystream.sections.personalization.inferred.locale-config", "en-US,en-GB,en-CA");
|
||||
|
||||
pref("browser.newtabpage.activity-stream.discoverystream.sections.personalization.inferred.user.enabled", true);
|
||||
// Override inferred personalization model JSON string that typically comes from rec API. Or "TEST" for a test model.
|
||||
pref("browser.newtabpage.activity-stream.discoverystream.sections.personalization.inferred.model.override", "");
|
||||
|
||||
pref("browser.newtabpage.activity-stream.discoverystream.sections.interestPicker.enabled", false);
|
||||
pref("browser.newtabpage.activity-stream.discoverystream.sections.interestPicker.visibleSections", "");
|
||||
|
||||
@@ -97,6 +97,7 @@ for (const type of [
|
||||
"HIDE_PERSONALIZE",
|
||||
"HIDE_PRIVACY_INFO",
|
||||
"HIDE_TOAST_MESSAGE",
|
||||
"INFERRED_PERSONALIZATION_MODEL_UPDATE",
|
||||
"INFERRED_PERSONALIZATION_REFRESH",
|
||||
"INFERRED_PERSONALIZATION_RESET",
|
||||
"INFERRED_PERSONALIZATION_UPDATE",
|
||||
|
||||
@@ -397,6 +397,17 @@ export class DiscoveryStreamAdminUI extends React.PureComponent {
|
||||
return weatherTable;
|
||||
}
|
||||
|
||||
renderPersonalizationData() {
|
||||
const { interestVector } = this.props.state.InferredPersonalization;
|
||||
return (
|
||||
<div>
|
||||
{" "}
|
||||
Interest Vector:
|
||||
<pre>{JSON.stringify(interestVector, null, 2)}</pre>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
renderFeedData(url) {
|
||||
const { feeds } = this.props.state.DiscoveryStream;
|
||||
const feed = feeds.data[url].data;
|
||||
@@ -723,6 +734,8 @@ export class DiscoveryStreamAdminUI extends React.PureComponent {
|
||||
<div className="large-data-container">{this.renderBlocksData()}</div>
|
||||
<h3>Weather Data</h3>
|
||||
{this.renderWeatherData()}
|
||||
<h3>Personalization Data</h3>
|
||||
{this.renderPersonalizationData()}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -760,6 +773,7 @@ export class DiscoveryStreamAdminInner extends React.PureComponent {
|
||||
DiscoveryStream: this.props.DiscoveryStream,
|
||||
Personalization: this.props.Personalization,
|
||||
Weather: this.props.Weather,
|
||||
InferredPersonalization: this.props.InferredPersonalization,
|
||||
}}
|
||||
otherPrefs={this.props.Prefs.values}
|
||||
dispatch={this.props.dispatch}
|
||||
@@ -847,6 +861,7 @@ export const DiscoveryStreamAdmin = connect(state => ({
|
||||
Sections: state.Sections,
|
||||
DiscoveryStream: state.DiscoveryStream,
|
||||
Personalization: state.Personalization,
|
||||
InferredPersonalization: state.InferredPersonalization,
|
||||
Prefs: state.Prefs,
|
||||
Weather: state.Weather,
|
||||
}))(_DiscoveryStreamAdmin);
|
||||
|
||||
@@ -170,6 +170,7 @@ for (const type of [
|
||||
"HIDE_PERSONALIZE",
|
||||
"HIDE_PRIVACY_INFO",
|
||||
"HIDE_TOAST_MESSAGE",
|
||||
"INFERRED_PERSONALIZATION_MODEL_UPDATE",
|
||||
"INFERRED_PERSONALIZATION_REFRESH",
|
||||
"INFERRED_PERSONALIZATION_RESET",
|
||||
"INFERRED_PERSONALIZATION_UPDATE",
|
||||
@@ -932,6 +933,12 @@ class DiscoveryStreamAdminUI extends (external_React_default()).PureComponent {
|
||||
}
|
||||
return weatherTable;
|
||||
}
|
||||
renderPersonalizationData() {
|
||||
const {
|
||||
interestVector
|
||||
} = this.props.state.InferredPersonalization;
|
||||
return /*#__PURE__*/external_React_default().createElement("div", null, " ", "Interest Vector:", /*#__PURE__*/external_React_default().createElement("pre", null, JSON.stringify(interestVector, null, 2)));
|
||||
}
|
||||
renderFeedData(url) {
|
||||
const {
|
||||
feeds
|
||||
@@ -1135,7 +1142,7 @@ class DiscoveryStreamAdminUI extends (external_React_default()).PureComponent {
|
||||
className: "large-data-container"
|
||||
}, this.renderImpressionsData()), /*#__PURE__*/external_React_default().createElement("h3", null, "Blocked Data"), /*#__PURE__*/external_React_default().createElement("div", {
|
||||
className: "large-data-container"
|
||||
}, this.renderBlocksData()), /*#__PURE__*/external_React_default().createElement("h3", null, "Weather Data"), this.renderWeatherData());
|
||||
}, this.renderBlocksData()), /*#__PURE__*/external_React_default().createElement("h3", null, "Weather Data"), this.renderWeatherData(), /*#__PURE__*/external_React_default().createElement("h3", null, "Personalization Data"), this.renderPersonalizationData());
|
||||
}
|
||||
}
|
||||
class DiscoveryStreamAdminInner extends (external_React_default()).PureComponent {
|
||||
@@ -1159,7 +1166,8 @@ class DiscoveryStreamAdminInner extends (external_React_default()).PureComponent
|
||||
state: {
|
||||
DiscoveryStream: this.props.DiscoveryStream,
|
||||
Personalization: this.props.Personalization,
|
||||
Weather: this.props.Weather
|
||||
Weather: this.props.Weather,
|
||||
InferredPersonalization: this.props.InferredPersonalization
|
||||
},
|
||||
otherPrefs: this.props.Prefs.values,
|
||||
dispatch: this.props.dispatch
|
||||
@@ -1229,6 +1237,7 @@ const DiscoveryStreamAdmin = (0,external_ReactRedux_namespaceObject.connect)(sta
|
||||
Sections: state.Sections,
|
||||
DiscoveryStream: state.DiscoveryStream,
|
||||
Personalization: state.Personalization,
|
||||
InferredPersonalization: state.InferredPersonalization,
|
||||
Prefs: state.Prefs,
|
||||
Weather: state.Weather
|
||||
}))(_DiscoveryStreamAdmin);
|
||||
|
||||
@@ -674,6 +674,13 @@ export const PREFS_CONFIG = new Map([
|
||||
value: false,
|
||||
},
|
||||
],
|
||||
[
|
||||
"discoverystream.sections.personalization.inferred.model.override",
|
||||
{
|
||||
title:
|
||||
"Override inferred personalization model JSON string that typically comes from rec API. Or 'TEST' for a test model",
|
||||
},
|
||||
],
|
||||
[
|
||||
"discoverystream.sections.cards.thumbsUpDown.enabled",
|
||||
{
|
||||
|
||||
@@ -1866,7 +1866,14 @@ export class DiscoveryStreamFeed {
|
||||
return { sectionId, title };
|
||||
});
|
||||
}
|
||||
|
||||
if (feedResponse.inferredLocalModel) {
|
||||
this.store.dispatch(
|
||||
ac.AlsoToMain({
|
||||
type: at.INFERRED_PERSONALIZATION_MODEL_UPDATE,
|
||||
data: feedResponse.inferredLocalModel || {},
|
||||
})
|
||||
);
|
||||
}
|
||||
// We can cleanup any impressions we have that are old before we rotate.
|
||||
// In theory we can do this anywhere, but doing it just before rotate is optimal.
|
||||
// Rotate is also the only place that uses these impressions.
|
||||
@@ -2877,13 +2884,15 @@ export class DiscoveryStreamFeed {
|
||||
break;
|
||||
case at.SECTION_PERSONALIZATION_SET:
|
||||
await this.cache.set("sectionPersonalization", action.data);
|
||||
|
||||
this.store.dispatch(
|
||||
ac.BroadcastToContent({
|
||||
type: at.SECTION_PERSONALIZATION_UPDATE,
|
||||
data: action.data,
|
||||
})
|
||||
);
|
||||
break;
|
||||
case at.INFERRED_PERSONALIZATION_MODEL_UPDATE:
|
||||
await this.cache.set("inferred_model", action.data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
369
browser/extensions/newtab/lib/InferredModel/FeatureModel.sys.mjs
Normal file
369
browser/extensions/newtab/lib/InferredModel/FeatureModel.sys.mjs
Normal file
@@ -0,0 +1,369 @@
|
||||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
/**
|
||||
* This file has classes to combine New Tab feature events (aggregated from a sqlLite table) into an interest model.
|
||||
*/
|
||||
|
||||
import {
|
||||
FORMAT,
|
||||
AggregateResultKeys,
|
||||
SPECIAL_FEATURE_CLICK,
|
||||
} from "resource://newtab/lib/InferredModel/InferredConstants.sys.mjs";
|
||||
|
||||
export const DAYS_TO_MS = 60 * 60 * 24 * 1000;
|
||||
|
||||
const MAX_INT_32 = 2 ** 32;
|
||||
|
||||
/**
|
||||
* Unary encoding with randomized response for differential privacy.
|
||||
* The output must be decoded to back to an integer when aggregating a historgram on a server
|
||||
* @param {number} x - Integer input (0 <= x < N)
|
||||
* @param {number} N - Number of values (see ablove)
|
||||
* @param {number} p - Probability of keeping a 1-bit as 1 (after one-hot encoding the output)
|
||||
* @param {number} q - Probability of flipping a 0-bit to 1
|
||||
* @returns {string} - Bitstring after unary encoding and randomized response
|
||||
*/
|
||||
export function unaryEncodeDiffPrivacy(x, N, p, q) {
|
||||
const bitstring = [];
|
||||
const randomValues = new Uint32Array(N);
|
||||
crypto.getRandomValues(randomValues);
|
||||
for (let i = 0; i < N; i++) {
|
||||
const trueBit = i === x ? 1 : 0;
|
||||
const rand = randomValues[i] / MAX_INT_32;
|
||||
if (trueBit === 1) {
|
||||
bitstring.push(rand <= p ? "1" : "0");
|
||||
} else {
|
||||
bitstring.push(rand <= q ? "1" : "0");
|
||||
}
|
||||
}
|
||||
return bitstring.join("");
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds value to all a particular key in a dictionary. If the key is missing it sets the value.
|
||||
* @param {Object} dict - The dictionary to modify.
|
||||
* @param {string} key - The key whose value should be added or set.
|
||||
* @param {number} value - The value to add to the key.
|
||||
*/
|
||||
export function dictAdd(dict, key, value) {
|
||||
if (key in dict) {
|
||||
dict[key] += value;
|
||||
} else {
|
||||
dict[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply function to all keys in dictionary, returning new dictionary.
|
||||
* @param {Object} obj - The object whose values should be transformed.
|
||||
* @param {Function} fn - The function to apply to each value.
|
||||
* @returns {Object} A new object with the transformed values.
|
||||
*/
|
||||
export function dictApply(obj, fn) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(obj).map(([key, value]) => [key, fn(value)])
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Class for re-scaling events based on time passed.
|
||||
*/
|
||||
export class DayTimeWeighting {
|
||||
/**
|
||||
* Instantiate class based on a series of day periods in the past.
|
||||
* @param {int[]} pastDays Series of number of days, indicating days ago intervals in reverse chonological order.
|
||||
* Intervals are added: If the first value is 1 and the second is 5, then the first inteval is 0-1 and second interval is between 1 and 6.
|
||||
* @param {number[]} relativeWeight Relative weight of each period. Must be same length as pastDays
|
||||
*/
|
||||
constructor(pastDays, relativeWeight) {
|
||||
this.pastDays = pastDays;
|
||||
this.relativeWeight = relativeWeight;
|
||||
}
|
||||
|
||||
static fromJSON(json) {
|
||||
return new DayTimeWeighting(json.days, json.relative_weight);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a series of interval pairs in the past based on the pastDays.
|
||||
* @param {number} curTimeMs Base time time in MS. Usually current time.
|
||||
* @returns
|
||||
*/
|
||||
getDateIntervals(curTimeMs) {
|
||||
let curEndTime = curTimeMs;
|
||||
|
||||
const res = this.pastDays.map(daysAgo => {
|
||||
const start = new Date(curEndTime - daysAgo * DAYS_TO_MS);
|
||||
const end = new Date(curEndTime);
|
||||
|
||||
curEndTime = start;
|
||||
return { start, end };
|
||||
});
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get relative weight of current index.
|
||||
* @param {int} weightIndex Index
|
||||
* @returns {number} Weight at index, or 0 if index out of range.
|
||||
*/
|
||||
getRelativeWeight(weightIndex) {
|
||||
if (weightIndex >= this.pastDays.length) {
|
||||
return 0;
|
||||
}
|
||||
return this.relativeWeight[weightIndex];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Describes the mapping from a set of aggregated events to a single interest feature
|
||||
*/
|
||||
export class InterestFeatures {
|
||||
constructor(
|
||||
name,
|
||||
featureWeights,
|
||||
thresholds = null,
|
||||
diff_p = 0.5,
|
||||
diff_q = 0.5
|
||||
) {
|
||||
this.name = name;
|
||||
this.featureWeights = featureWeights;
|
||||
// Thresholds must be in ascending order
|
||||
this.thresholds = thresholds;
|
||||
this.diff_p = diff_p;
|
||||
this.diff_q = diff_q;
|
||||
}
|
||||
|
||||
static fromJSON(name, json) {
|
||||
return new InterestFeatures(
|
||||
name,
|
||||
json.features,
|
||||
json.thresholds || null,
|
||||
json.diff_p,
|
||||
json.diff_q
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Quantize a feature value based on the thresholds specified in the class.
|
||||
* @param {number} inValue Value computed by model for the feature.
|
||||
* @returns Quantized value. A value between 0 and number of thresholds specified (inclusive)
|
||||
*/
|
||||
applyThresholds(inValue) {
|
||||
if (!this.thresholds) {
|
||||
return inValue;
|
||||
}
|
||||
for (let k = 0; k < this.thresholds.length; k++) {
|
||||
if (inValue < this.thresholds[k]) {
|
||||
return k;
|
||||
}
|
||||
}
|
||||
return this.thresholds.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies Differential Privacy Unary Encoding method, outputting a one-hot encoded vector with randomizaiton.
|
||||
* Accurate historgrams of values can be computed with reasonable accuracy.
|
||||
* If the class has no or 0 p/q values set for differential privacy, then response is original number non-encoded.
|
||||
* @param {number} inValue Value to randomize
|
||||
* @returns Bitfield as a string, that is the same as the thresholds length + 1
|
||||
*/
|
||||
applyDifferentialPrivacy(inValue) {
|
||||
if (!this.thresholds || !this.diff_p) {
|
||||
return inValue;
|
||||
}
|
||||
return unaryEncodeDiffPrivacy(
|
||||
inValue,
|
||||
this.thresholds.length + 1,
|
||||
this.diff_p,
|
||||
this.diff_q
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Manages relative tile importance
|
||||
*/
|
||||
export class TileImportance {
|
||||
constructor(tileImportanceMappings) {
|
||||
this.mappings = {};
|
||||
for (const [formatKey, formatId] of Object.entries(FORMAT)) {
|
||||
if (formatKey in tileImportanceMappings) {
|
||||
this.mappings[formatId] = tileImportanceMappings[formatKey];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
getRelativeCTRForTile(tileType) {
|
||||
return this.mappings[tileType] || 1;
|
||||
}
|
||||
|
||||
static fromJSON(json) {
|
||||
return new TileImportance(json);
|
||||
}
|
||||
}
|
||||
|
||||
/***
|
||||
* A simple model for aggregating features
|
||||
*/
|
||||
|
||||
export class FeatureModel {
|
||||
/**
|
||||
*
|
||||
* @param {string} modelId
|
||||
* @param {Object} dayTimeWeighting Data for day time weighting class
|
||||
* @param {Object} interestVectorModel Data for interest model
|
||||
* @param {Object} tileImportance Data for tile importance
|
||||
* @param {boolean} rescale Whether to rescale to max value
|
||||
* @param {boolean} logScale Whether to apply natural log (ln(x+ 1)) before rescaling
|
||||
*/
|
||||
constructor({
|
||||
modelId,
|
||||
dayTimeWeighting,
|
||||
interestVectorModel,
|
||||
tileImportance,
|
||||
modelType,
|
||||
rescale = true,
|
||||
logScale = false,
|
||||
}) {
|
||||
this.modelId = modelId;
|
||||
this.tileImportance = tileImportance;
|
||||
this.dayTimeWeighting = dayTimeWeighting;
|
||||
this.interestVectorModel = interestVectorModel;
|
||||
this.rescale = rescale;
|
||||
this.logScale = logScale;
|
||||
this.modelType = modelType;
|
||||
}
|
||||
|
||||
static fromJSON(json) {
|
||||
const dayTimeWeighting = DayTimeWeighting.fromJSON(json.day_time_weighting);
|
||||
const interestVectorModel = {};
|
||||
const tileImportance = TileImportance.fromJSON(json.tile_importance || {});
|
||||
|
||||
for (const [name, featureJson] of Object.entries(json.interest_vector)) {
|
||||
interestVectorModel[name] = InterestFeatures.fromJSON(name, featureJson);
|
||||
}
|
||||
|
||||
return new FeatureModel({
|
||||
dayTimeWeighting,
|
||||
tileImportance,
|
||||
interestVectorModel,
|
||||
normalize: json.normalize,
|
||||
rescale: json.rescale,
|
||||
logScale: json.log_scale,
|
||||
clickScale: json.clickScale,
|
||||
modelType: json.model_type,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Return date intervals for the query
|
||||
*/
|
||||
getDateIntervals(curTimeMs) {
|
||||
return this.dayTimeWeighting.getDateIntervals(curTimeMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes an interest vector or aggregate based on the model and raw sql inout.
|
||||
* @param {Object} config
|
||||
* @param {Array.<Array.<string|number>>} config.dataForIntervals Raw aggregate output from SQL query. Could be clicks or impressions
|
||||
* @param {Object.<string, number>} config.indexSchema Map of keys to indices in each sub-array in dataForIntervals
|
||||
* @param {boolean} [config.applyThresholding=false] Whether to apply thresholds
|
||||
* @param {boolean} [config.applyDifferntialPrivacy=false] Whether to apply differential privacy. This will be used for sending to telemetry.
|
||||
* @returns
|
||||
*/
|
||||
computeInterestVector({
|
||||
dataForIntervals,
|
||||
indexSchema,
|
||||
applyThresholding = false,
|
||||
applyDifferentialPrivacy = false,
|
||||
}) {
|
||||
const processedPerTimeInterval = dataForIntervals.map(
|
||||
(intervalData, idx) => {
|
||||
const intervalRawTotal = {};
|
||||
const perPeriodTotals = {};
|
||||
intervalData.forEach(aggElement => {
|
||||
const feature = aggElement[indexSchema[AggregateResultKeys.FEATURE]];
|
||||
let value = aggElement[indexSchema[AggregateResultKeys.VALUE]]; // In the future we could support format here
|
||||
dictAdd(intervalRawTotal, feature, value);
|
||||
});
|
||||
|
||||
const weight = this.dayTimeWeighting.getRelativeWeight(idx); // Weight for this time interval
|
||||
Object.values(this.interestVectorModel).forEach(interestFeature => {
|
||||
for (const featureUsed of Object.keys(
|
||||
interestFeature.featureWeights
|
||||
)) {
|
||||
if (featureUsed in intervalRawTotal) {
|
||||
dictAdd(
|
||||
perPeriodTotals,
|
||||
interestFeature.name,
|
||||
intervalRawTotal[featureUsed] *
|
||||
weight *
|
||||
interestFeature.featureWeights[featureUsed]
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
return perPeriodTotals;
|
||||
}
|
||||
);
|
||||
|
||||
// Since we are doing linear combinations, it is fine to do the day-time weighting at this step
|
||||
let totalResults = {};
|
||||
processedPerTimeInterval.forEach(intervalTotals => {
|
||||
for (const key of Object.keys(intervalTotals)) {
|
||||
dictAdd(totalResults, key, intervalTotals[key]);
|
||||
}
|
||||
});
|
||||
|
||||
let numClicks = -1;
|
||||
|
||||
// If clicks is a feature, it's handled as special case
|
||||
if (SPECIAL_FEATURE_CLICK in totalResults) {
|
||||
numClicks = totalResults[SPECIAL_FEATURE_CLICK];
|
||||
delete totalResults[SPECIAL_FEATURE_CLICK];
|
||||
}
|
||||
|
||||
if (this.logScale) {
|
||||
totalResults = dictApply(totalResults, x => Math.log(x + 1));
|
||||
}
|
||||
|
||||
if (this.rescale) {
|
||||
let divisor = Math.max(...Object.values(totalResults));
|
||||
if (divisor <= 0.001) {
|
||||
divisor = 0.001;
|
||||
}
|
||||
totalResults = dictApply(totalResults, x => x / divisor);
|
||||
}
|
||||
|
||||
if (this.clickScale && numClicks > 0) {
|
||||
totalResults = dictApply(totalResults, x => x / numClicks);
|
||||
}
|
||||
|
||||
if (numClicks >= 0) {
|
||||
totalResults[SPECIAL_FEATURE_CLICK] = numClicks;
|
||||
}
|
||||
|
||||
if (applyThresholding) {
|
||||
for (const key of Object.keys(totalResults)) {
|
||||
if (key in this.interestVectorModel) {
|
||||
totalResults[key] = this.interestVectorModel[key].applyThresholds(
|
||||
totalResults[key],
|
||||
applyDifferentialPrivacy
|
||||
);
|
||||
if (applyDifferentialPrivacy) {
|
||||
totalResults[key] = this.interestVectorModel[
|
||||
key
|
||||
].applyDifferentialPrivacy(
|
||||
totalResults[key],
|
||||
applyDifferentialPrivacy
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return totalResults;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
export const FORMAT_ENUM = {
|
||||
SMALL: 0,
|
||||
MEDIUM: 1,
|
||||
LARGE: 2,
|
||||
};
|
||||
|
||||
export const FORMAT = {
|
||||
"small-card": FORMAT_ENUM.SMALL,
|
||||
"medium-card": FORMAT_ENUM.MEDIUM,
|
||||
"large-card": FORMAT_ENUM.LARGE,
|
||||
};
|
||||
|
||||
/**
|
||||
* We are exploring two options for interest vectors
|
||||
*/
|
||||
export const MODEL_TYPE = {
|
||||
// Returns clicks and impressions as separate dictionaries
|
||||
CLICK_IMP_PAIR: "click_impression_pair",
|
||||
// Returns a single clicks dictionary, along with the total number of clicks
|
||||
CLICKS: "clicks",
|
||||
};
|
||||
|
||||
export const CLICK_FEATURE = "click";
|
||||
|
||||
export const AggregateResultKeys = {
|
||||
POSITION: "position",
|
||||
FEATURE: "feature",
|
||||
VALUE: "feature_value",
|
||||
SECTION_POSITION: "section_position",
|
||||
FORMAT_ENUM: "card_format_enum",
|
||||
};
|
||||
|
||||
// Clicks feature is handled in certain ways by the model
|
||||
export const SPECIAL_FEATURE_CLICK = "clicks";
|
||||
|
||||
export const DEFAULT_INFERRED_MODEL_DATA = {
|
||||
model_type: MODEL_TYPE.CLICKS,
|
||||
rescale: true,
|
||||
day_time_weighting: {
|
||||
days: [3, 14, 45],
|
||||
relative_weight: [1, 1, 1],
|
||||
},
|
||||
interest_vector: {
|
||||
parenting: {
|
||||
features: { parenting: 1 },
|
||||
thresholds: [0.3, 0.4],
|
||||
diff_p: 0.75,
|
||||
diff_q: 0.25,
|
||||
},
|
||||
arts: {
|
||||
features: { arts: 1 },
|
||||
thresholds: [0.3, 0.4],
|
||||
diff_p: 0.75,
|
||||
diff_q: 0.25,
|
||||
},
|
||||
health: {
|
||||
features: { arts: 1 },
|
||||
thresholds: [0.3, 0.4],
|
||||
diff_p: 0.75,
|
||||
diff_q: 0.25,
|
||||
},
|
||||
sports: {
|
||||
features: { sports: 1 },
|
||||
thresholds: [0.3, 0.4],
|
||||
diff_p: 0.75,
|
||||
diff_q: 0.25,
|
||||
},
|
||||
society: {
|
||||
features: { society: 1 },
|
||||
thresholds: [0.3, 0.4],
|
||||
diff_p: 0.75,
|
||||
diff_q: 0.25,
|
||||
},
|
||||
education: {
|
||||
features: { education: 1 },
|
||||
thresholds: [0.3, 0.4],
|
||||
diff_p: 0.75,
|
||||
diff_q: 0.25,
|
||||
},
|
||||
government: {
|
||||
features: { government: 1 },
|
||||
thresholds: [0.3, 0.4],
|
||||
diff_p: 0.75,
|
||||
diff_q: 0.25,
|
||||
},
|
||||
[SPECIAL_FEATURE_CLICK]: {
|
||||
features: { click: 1 },
|
||||
thresholds: [2, 8, 40],
|
||||
diff_p: 0.9,
|
||||
diff_q: 0.1,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
export const DEFAULT_INFERRED_MODEL = {
|
||||
model_id: "default",
|
||||
model_data: DEFAULT_INFERRED_MODEL_DATA,
|
||||
};
|
||||
@@ -9,29 +9,38 @@ ChromeUtils.defineESModuleGetters(lazy, {
|
||||
PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
|
||||
});
|
||||
|
||||
import { FeatureModel } from "resource://newtab/lib/InferredModel/FeatureModel.sys.mjs";
|
||||
|
||||
import {
|
||||
FORMAT,
|
||||
AggregateResultKeys,
|
||||
DEFAULT_INFERRED_MODEL_DATA,
|
||||
} from "resource://newtab/lib/InferredModel/InferredConstants.sys.mjs";
|
||||
|
||||
import {
|
||||
actionTypes as at,
|
||||
actionCreators as ac,
|
||||
} from "resource://newtab/common/Actions.mjs";
|
||||
|
||||
import { MODEL_TYPE } from "./InferredModel/InferredConstants.sys.mjs";
|
||||
|
||||
const CACHE_KEY = "inferred_personalization_feed";
|
||||
const DISCOVERY_STREAM_CACHE_KEY = "discovery_stream";
|
||||
const INTEREST_VECTOR_UPDATE_TIME = 4 * 60 * 60 * 1000; // 4 hours
|
||||
const PREF_USER_INFERRED_PERSONALIZATION =
|
||||
"discoverystream.sections.personalization.inferred.user.enabled";
|
||||
const PREF_SYSTEM_INFERRED_PERSONALIZATION =
|
||||
"discoverystream.sections.personalization.inferred.enabled";
|
||||
const PREF_SYSTEM_INFERRED_MODEL_OVERRIDE =
|
||||
"discoverystream.sections.personalization.inferred.model.override";
|
||||
|
||||
const FORMAT_ENUM = {
|
||||
SMALL: 0,
|
||||
MEDIUM: 1,
|
||||
LARGE: 2,
|
||||
};
|
||||
function timeMSToSeconds(timeMS) {
|
||||
return Math.round(timeMS / 1000);
|
||||
}
|
||||
|
||||
const FORMAT = {
|
||||
"small-card": FORMAT_ENUM.SMALL,
|
||||
"medium-card": FORMAT_ENUM.MEDIUM,
|
||||
"large-card": FORMAT_ENUM.LARGE,
|
||||
};
|
||||
const CLICK_TABLE = "moz_newtab_story_click";
|
||||
const IMPRESSION_TABLE = "moz_newtab_story_impression";
|
||||
const TEST_MODEL_ID = "TEST";
|
||||
|
||||
/**
|
||||
* A feature that periodically generates a interest vector for inferred personalization.
|
||||
@@ -65,9 +74,88 @@ export class InferredPersonalizationFeed {
|
||||
await this.loadInterestVector(true /* isStartup */);
|
||||
}
|
||||
|
||||
async queryDatabaseForTimeIntervals(intervals, table) {
|
||||
let results = [];
|
||||
for (const interval of intervals) {
|
||||
const agg = await this.fetchInferredPersonalizationSummary(
|
||||
interval.start,
|
||||
interval.end,
|
||||
table
|
||||
);
|
||||
results.push(agg);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Inferrred model raw data
|
||||
* @returns JSON of inferred model
|
||||
*/
|
||||
async getInferredModelData() {
|
||||
const modelOverrideRaw =
|
||||
this.store.getState().Prefs.values[PREF_SYSTEM_INFERRED_MODEL_OVERRIDE];
|
||||
if (modelOverrideRaw) {
|
||||
if (modelOverrideRaw === TEST_MODEL_ID) {
|
||||
return {
|
||||
model_id: TEST_MODEL_ID,
|
||||
model_data: DEFAULT_INFERRED_MODEL_DATA,
|
||||
};
|
||||
}
|
||||
try {
|
||||
return JSON.parse(modelOverrideRaw);
|
||||
} catch (_error) {}
|
||||
}
|
||||
const dsCache = this.PersistentCache(DISCOVERY_STREAM_CACHE_KEY, true);
|
||||
const cachedData = (await dsCache.get()) || {};
|
||||
let { inferredModel } = cachedData;
|
||||
return inferredModel;
|
||||
}
|
||||
|
||||
async generateInterestVector() {
|
||||
// TODO items and model should be props passed in, or fetched in this function.
|
||||
// TODO Run items and model to generate interest vector.
|
||||
const inferredModel = await this.getInferredModelData();
|
||||
if (!inferredModel || !inferredModel.model_data) {
|
||||
return {};
|
||||
}
|
||||
const model = FeatureModel.fromJSON(inferredModel.model_data);
|
||||
|
||||
const intervals = model.getDateIntervals(this.Date().now());
|
||||
const schema = {
|
||||
[AggregateResultKeys.FEATURE]: 0,
|
||||
[AggregateResultKeys.FORMAT_ENUM]: 1,
|
||||
[AggregateResultKeys.VALUE]: 2,
|
||||
};
|
||||
|
||||
const aggClickPerInterval = await this.queryDatabaseForTimeIntervals(
|
||||
intervals,
|
||||
CLICK_TABLE
|
||||
);
|
||||
|
||||
const ivClicks = model.computeInterestVector({
|
||||
dataForIntervals: aggClickPerInterval,
|
||||
indexSchema: schema,
|
||||
});
|
||||
|
||||
if (model.modelType === MODEL_TYPE.CLICKS) {
|
||||
return { ...ivClicks, model_id: inferredModel.model_id };
|
||||
}
|
||||
|
||||
if (model.modelType === MODEL_TYPE.CLICK_IMP_PAIR) {
|
||||
const aggImpressionsPerInterval =
|
||||
await this.queryDatabaseForTimeIntervals(intervals, IMPRESSION_TABLE);
|
||||
const ivImpressions = model.computeInterestVector({
|
||||
dataForIntervals: aggImpressionsPerInterval,
|
||||
indexSchema: schema,
|
||||
});
|
||||
const res = {
|
||||
c: ivClicks,
|
||||
i: ivImpressions,
|
||||
model_id: inferredModel.model_id,
|
||||
};
|
||||
return res;
|
||||
}
|
||||
|
||||
// unsupported modelType
|
||||
return {};
|
||||
}
|
||||
|
||||
async loadInterestVector(isStartup = false) {
|
||||
@@ -82,6 +170,7 @@ export class InferredPersonalizationFeed {
|
||||
INTEREST_VECTOR_UPDATE_TIME
|
||||
)
|
||||
) {
|
||||
// TODO Get model from Merino/DiscoveryStreamFeed
|
||||
interest_vector = {
|
||||
data: await this.generateInterestVector(),
|
||||
lastUpdated: this.Date().now(),
|
||||
@@ -89,6 +178,7 @@ export class InferredPersonalizationFeed {
|
||||
}
|
||||
await this.cache.set("interest_vector", interest_vector);
|
||||
this.loaded = true;
|
||||
|
||||
this.store.dispatch(
|
||||
ac.OnlyToMain({
|
||||
type: at.INFERRED_PERSONALIZATION_UPDATE,
|
||||
@@ -142,15 +232,13 @@ export class InferredPersonalizationFeed {
|
||||
}
|
||||
|
||||
async recordInferredPersonalizationImpression(tile) {
|
||||
await this.recordInferredPersonalizationInteraction(
|
||||
"moz_newtab_story_impression",
|
||||
tile
|
||||
);
|
||||
await this.recordInferredPersonalizationInteraction(IMPRESSION_TABLE, tile);
|
||||
}
|
||||
async recordInferredPersonalizationClick(tile) {
|
||||
await this.recordInferredPersonalizationInteraction(
|
||||
"moz_newtab_story_click",
|
||||
tile
|
||||
CLICK_TABLE,
|
||||
tile,
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
@@ -159,47 +247,60 @@ export class InferredPersonalizationFeed {
|
||||
"moz_newtab_story_impression"
|
||||
);
|
||||
}
|
||||
async fetchInferredPersonalizationClick() {
|
||||
return await this.fetchInferredPersonalizationInteraction(
|
||||
"moz_newtab_story_click"
|
||||
);
|
||||
|
||||
async fetchInferredPersonalizationSummary(startTime, endTime, table) {
|
||||
let sql = `SELECT feature, card_format_enum, SUM(feature_value) FROM ${table}
|
||||
WHERE timestamp_s > ${timeMSToSeconds(startTime)}
|
||||
AND timestamp_s < ${timeMSToSeconds(endTime)}
|
||||
GROUP BY feature, card_format_enum`;
|
||||
const { activityStreamProvider } = lazy.NewTabUtils;
|
||||
const interactions = await activityStreamProvider.executePlacesQuery(sql);
|
||||
return interactions;
|
||||
}
|
||||
|
||||
async recordInferredPersonalizationInteraction(table, tile) {
|
||||
const feature = tile.topic;
|
||||
const timestamp_s = this.Date().now() / 1000;
|
||||
async recordInferredPersonalizationInteraction(
|
||||
table,
|
||||
tile,
|
||||
extraClickEvent = false
|
||||
) {
|
||||
const timestamp_s = timeMSToSeconds(this.Date().now());
|
||||
const card_format_enum = FORMAT[tile.format];
|
||||
const position = tile.pos;
|
||||
const section_position = tile.section_position || 0;
|
||||
// TODO This needs to be attached to the tile, and coming from Merino.
|
||||
// TODO This is now in tile.features.
|
||||
// TODO It may be undefined if previous data was cached before Merino started returning features.
|
||||
const feature_value = 0.5;
|
||||
|
||||
if (
|
||||
table !== "moz_newtab_story_impression" &&
|
||||
table !== "moz_newtab_story_click"
|
||||
) {
|
||||
let featureValuePairs = [];
|
||||
if (extraClickEvent) {
|
||||
featureValuePairs.push(["click", 1]);
|
||||
}
|
||||
if (tile.features) {
|
||||
featureValuePairs = featureValuePairs.concat(
|
||||
Object.entries(tile.features)
|
||||
);
|
||||
}
|
||||
if (table !== CLICK_TABLE && table !== IMPRESSION_TABLE) {
|
||||
return;
|
||||
}
|
||||
const primaryValues = {
|
||||
timestamp_s,
|
||||
card_format_enum,
|
||||
position,
|
||||
section_position,
|
||||
};
|
||||
|
||||
const insertValues = featureValuePairs.map(pair =>
|
||||
Object.assign({}, primaryValues, {
|
||||
feature: pair[0],
|
||||
feature_value: pair[1],
|
||||
})
|
||||
);
|
||||
|
||||
let sql = `
|
||||
INSERT INTO ${table}(feature, timestamp_s, card_format_enum, position, section_position, feature_value)
|
||||
VALUES (:feature, :timestamp_s, :card_format_enum, :position, :section_position, :feature_value)
|
||||
`;
|
||||
await lazy.PlacesUtils.withConnectionWrapper(
|
||||
"newtab/lib/TelemetryFeed.sys.mjs: recordInferredPersonalizationImpression",
|
||||
"newtab/lib/InferredPersonalizationFeed.sys.mjs: recordInferredPersonalizationImpression",
|
||||
async db => {
|
||||
await db.execute(
|
||||
`
|
||||
INSERT INTO ${table}(feature, timestamp_s, card_format_enum, position, section_position, feature_value)
|
||||
VALUES (:feature, :timestamp_s, :card_format_enum, :position, :section_position, :feature_value)
|
||||
`,
|
||||
{
|
||||
feature,
|
||||
timestamp_s,
|
||||
card_format_enum,
|
||||
position,
|
||||
section_position,
|
||||
feature_value,
|
||||
}
|
||||
);
|
||||
await db.execute(sql, insertValues);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@@ -77,6 +77,9 @@ describe("DiscoveryStreamAdmin", () => {
|
||||
Weather: {
|
||||
suggestions: [],
|
||||
},
|
||||
InferredPersonalization: {
|
||||
interestVector: {},
|
||||
},
|
||||
}}
|
||||
/>
|
||||
);
|
||||
@@ -108,6 +111,9 @@ describe("DiscoveryStreamAdmin", () => {
|
||||
Weather: {
|
||||
suggestions: [],
|
||||
},
|
||||
InferredPersonalization: {
|
||||
interestVector: {},
|
||||
},
|
||||
}}
|
||||
/>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { FeatureModel } from "lib/InferredModel/FeatureModel.sys.mjs";
|
||||
|
||||
const jsonData = {
|
||||
model_id: "test",
|
||||
schema_ver: 1,
|
||||
day_time_weighting: {
|
||||
days: [3, 14, 45],
|
||||
relative_weight: [0.33, 0.33, 0.33],
|
||||
},
|
||||
interest_vector: {
|
||||
cryptosport: {
|
||||
features: { crypto: 0.5, sport: 0.5 },
|
||||
thresholds: [0.3, 0.4, 0.5],
|
||||
},
|
||||
parenting: {
|
||||
features: { parenting: 1 },
|
||||
thresholds: [0.3, 0.4],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
describe("Inferred Model", () => {
|
||||
it("create model", () => {
|
||||
const model = FeatureModel.fromJSON(jsonData);
|
||||
assert.equal(model.model_id, jsonData.model_id);
|
||||
});
|
||||
it("create time intervals", () => {
|
||||
const model = FeatureModel.fromJSON(jsonData);
|
||||
assert.equal(model.model_id, jsonData.model_id);
|
||||
const intervals = model.getDateIntervals();
|
||||
|
||||
const curTime = new Date();
|
||||
assert.equal(intervals.length, jsonData.day_time_weighting.days.length);
|
||||
for (const interval of intervals) {
|
||||
assert.isTrue(interval.start < curTime.getTime());
|
||||
assert.isTrue(interval.end <= curTime.getTime());
|
||||
assert.isTrue(interval.start <= interval.end);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,314 @@
|
||||
"use strict";
|
||||
|
||||
ChromeUtils.defineESModuleGetters(this, {
|
||||
FeatureModel: "resource://newtab/lib/InferredModel/FeatureModel.sys.mjs",
|
||||
dictAdd: "resource://newtab/lib/InferredModel/FeatureModel.sys.mjs",
|
||||
dictApply: "resource://newtab/lib/InferredModel/FeatureModel.sys.mjs",
|
||||
DayTimeWeighting: "resource://newtab/lib/InferredModel/FeatureModel.sys.mjs",
|
||||
InterestFeatures: "resource://newtab/lib/InferredModel/FeatureModel.sys.mjs",
|
||||
unaryEncodeDiffPrivacy:
|
||||
"resource://newtab/lib/InferredModel/FeatureModel.sys.mjs",
|
||||
});
|
||||
|
||||
add_task(function test_dictAdd() {
|
||||
let dict = {};
|
||||
dictAdd(dict, "a", 3);
|
||||
Assert.equal(dict.a, 3, "Should set value when key is missing");
|
||||
|
||||
dictAdd(dict, "a", 2);
|
||||
Assert.equal(dict.a, 5, "Should add value when key exists");
|
||||
});
|
||||
|
||||
add_task(function test_dictApply() {
|
||||
let input = { a: 1, b: 2 };
|
||||
let output = dictApply(input, x => x * 2);
|
||||
Assert.deepEqual(output, { a: 2, b: 4 }, "Should double all values");
|
||||
|
||||
let identity = dictApply(input, x => x);
|
||||
Assert.deepEqual(
|
||||
identity,
|
||||
input,
|
||||
"Should return same values with identity function"
|
||||
);
|
||||
});
|
||||
|
||||
add_task(function test_DayTimeWeighting_getDateIntervals() {
|
||||
let weighting = new DayTimeWeighting([1, 2], [0.5, 0.2]);
|
||||
let now = Date.now();
|
||||
let intervals = weighting.getDateIntervals(now);
|
||||
|
||||
Assert.equal(
|
||||
intervals.length,
|
||||
2,
|
||||
"Should return one interval per pastDay entry"
|
||||
);
|
||||
Assert.ok(
|
||||
intervals[0].end <= new Date(now),
|
||||
"Each interval end should be before or equal to now"
|
||||
);
|
||||
Assert.ok(
|
||||
intervals[0].start < intervals[0].end,
|
||||
"Start should be before end"
|
||||
);
|
||||
Assert.ok(
|
||||
intervals[1].end <= new Date(now),
|
||||
"Each interval end should be before or equal to now"
|
||||
);
|
||||
Assert.ok(
|
||||
intervals[1].start < intervals[0].end,
|
||||
"Start should be before end"
|
||||
);
|
||||
});
|
||||
|
||||
add_task(function test_DayTimeWeighting_getRelativeWeight() {
|
||||
let weighting = new DayTimeWeighting([1, 2], [0.5, 0.2]);
|
||||
|
||||
Assert.equal(
|
||||
weighting.getRelativeWeight(0),
|
||||
0.5,
|
||||
"Should return correct weight for index 0"
|
||||
);
|
||||
Assert.equal(
|
||||
weighting.getRelativeWeight(1),
|
||||
0.2,
|
||||
"Should return correct weight for index 1"
|
||||
);
|
||||
Assert.equal(
|
||||
weighting.getRelativeWeight(2),
|
||||
0,
|
||||
"Should return 0 for out-of-range index"
|
||||
);
|
||||
});
|
||||
|
||||
add_task(function test_DayTimeWeighting_fromJSON() {
|
||||
const json = { days: [1, 2], relative_weight: [0.1, 0.3] };
|
||||
const weighting = DayTimeWeighting.fromJSON(json);
|
||||
|
||||
Assert.ok(
|
||||
weighting instanceof DayTimeWeighting,
|
||||
"Should create instance from JSON"
|
||||
);
|
||||
Assert.deepEqual(
|
||||
weighting.pastDays,
|
||||
[1, 2],
|
||||
"Should correctly parse pastDays"
|
||||
);
|
||||
Assert.deepEqual(
|
||||
weighting.relativeWeight,
|
||||
[0.1, 0.3],
|
||||
"Should correctly parse relative weights"
|
||||
);
|
||||
});
|
||||
|
||||
add_task(function test_InterestFeatures_applyThresholds() {
|
||||
let feature = new InterestFeatures("test", {}, [10, 20, 30]);
|
||||
// Note that number of output is 1 + the length of the input weights
|
||||
Assert.equal(
|
||||
feature.applyThresholds(5),
|
||||
0,
|
||||
"Value < first threshold returns 0"
|
||||
);
|
||||
Assert.equal(
|
||||
feature.applyThresholds(15),
|
||||
1,
|
||||
"Value < second threshold returns 1"
|
||||
);
|
||||
Assert.equal(
|
||||
feature.applyThresholds(25),
|
||||
2,
|
||||
"Value < third threshold returns 2"
|
||||
);
|
||||
Assert.equal(
|
||||
feature.applyThresholds(35),
|
||||
3,
|
||||
"Value >= all thresholds returns length of thresholds"
|
||||
);
|
||||
});
|
||||
|
||||
add_task(function test_InterestFeatures_noThresholds() {
|
||||
let feature = new InterestFeatures("test", {});
|
||||
Assert.equal(
|
||||
feature.applyThresholds(42),
|
||||
42,
|
||||
"Without thresholds, should return input unchanged"
|
||||
);
|
||||
});
|
||||
|
||||
add_task(function test_InterestFeatures_fromJSON() {
|
||||
const json = { features: { a: 1 }, thresholds: [1, 2] };
|
||||
const feature = InterestFeatures.fromJSON("f", json);
|
||||
|
||||
Assert.ok(
|
||||
feature instanceof InterestFeatures,
|
||||
"Should create InterestFeatures from JSON"
|
||||
);
|
||||
Assert.equal(feature.name, "f", "Should set correct name");
|
||||
Assert.deepEqual(
|
||||
feature.featureWeights,
|
||||
{ a: 1 },
|
||||
"Should set correct feature weights"
|
||||
);
|
||||
Assert.deepEqual(feature.thresholds, [1, 2], "Should set correct thresholds");
|
||||
});
|
||||
|
||||
const SPECIAL_FEATURE_CLICK = "clicks";
|
||||
|
||||
const AggregateResultKeys = {
|
||||
POSITION: "position",
|
||||
FEATURE: "feature",
|
||||
VALUE: "feature_value",
|
||||
SECTION_POSITION: "section_position",
|
||||
FORMAT_ENUM: "card_format_enum",
|
||||
};
|
||||
|
||||
const SCHEMA = {
|
||||
[AggregateResultKeys.FEATURE]: 0,
|
||||
[AggregateResultKeys.FORMAT_ENUM]: 1,
|
||||
[AggregateResultKeys.VALUE]: 2,
|
||||
};
|
||||
|
||||
const jsonModelData = {
|
||||
model_type: "clicks",
|
||||
day_time_weighting: {
|
||||
days: [3, 14, 45],
|
||||
relative_weight: [1, 0.5, 0.3],
|
||||
},
|
||||
interest_vector: {
|
||||
news_reader: {
|
||||
features: { pub_nytimes_com: 0.5, pub_cnn_com: 0.5 },
|
||||
thresholds: [0.3, 0.4, 0.5],
|
||||
diff_p: 1,
|
||||
diff_q: 0,
|
||||
},
|
||||
parenting: {
|
||||
features: { parenting: 1 },
|
||||
thresholds: [0.3, 0.4],
|
||||
diff_p: 1,
|
||||
diff_q: 0,
|
||||
},
|
||||
[SPECIAL_FEATURE_CLICK]: {
|
||||
features: { click: 1 },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
add_task(function test_FeatureModel_fromJSON() {
|
||||
const model = FeatureModel.fromJSON(jsonModelData);
|
||||
const curTime = new Date();
|
||||
const intervals = model.getDateIntervals(curTime);
|
||||
Assert.equal(intervals.length, jsonModelData.day_time_weighting.days.length);
|
||||
for (const interval of intervals) {
|
||||
Assert.ok(
|
||||
interval.start.getTime() <= interval.end.getTime(),
|
||||
"Interval start and end are in correct order"
|
||||
);
|
||||
Assert.ok(
|
||||
interval.end.getTime() <= curTime.getTime(),
|
||||
"Interval end is not in future"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
const SQL_RESULT_DATA = [
|
||||
[
|
||||
["click", 0, 1],
|
||||
["parenting", 0, 1],
|
||||
],
|
||||
[
|
||||
["click", 0, 2],
|
||||
["parenting", 0, 1],
|
||||
["pub_nytimes_com", 0, 1],
|
||||
],
|
||||
[],
|
||||
];
|
||||
|
||||
add_task(function test_computeInterestVector() {
|
||||
const modelData = { ...jsonModelData, rescale: true };
|
||||
const model = FeatureModel.fromJSON(modelData);
|
||||
const result = model.computeInterestVector({
|
||||
dataForIntervals: SQL_RESULT_DATA,
|
||||
indexSchema: SCHEMA,
|
||||
applyThresholding: false,
|
||||
});
|
||||
Assert.ok("parenting" in result, "Result should contain parenting");
|
||||
Assert.ok("news_reader" in result, "Result should contain news_reader");
|
||||
Assert.equal(result.parenting, 1.0, "Vector is rescaled");
|
||||
|
||||
Assert.equal(
|
||||
result[SPECIAL_FEATURE_CLICK],
|
||||
2,
|
||||
"Should include rescaled raw click"
|
||||
);
|
||||
});
|
||||
|
||||
add_task(function test_computeThresholds() {
|
||||
const modelData = { ...jsonModelData, rescale: true };
|
||||
const model = FeatureModel.fromJSON(modelData);
|
||||
const result = model.computeInterestVector({
|
||||
dataForIntervals: SQL_RESULT_DATA,
|
||||
indexSchema: SCHEMA,
|
||||
applyThresholding: true,
|
||||
});
|
||||
Assert.equal(result.parenting, 2, "Threshold is applied");
|
||||
Assert.equal(
|
||||
result[SPECIAL_FEATURE_CLICK],
|
||||
2,
|
||||
"Should include rescaled raw click"
|
||||
);
|
||||
});
|
||||
|
||||
add_task(function test_unaryEncoding() {
|
||||
const numValues = 4;
|
||||
|
||||
Assert.equal(
|
||||
unaryEncodeDiffPrivacy(0, numValues, 1, 0),
|
||||
"1000",
|
||||
"Basic dp works with out of range p, q"
|
||||
);
|
||||
Assert.equal(
|
||||
unaryEncodeDiffPrivacy(1, numValues, 1, 0),
|
||||
"0100",
|
||||
"Basic dp works with out of range p, q"
|
||||
);
|
||||
Assert.equal(
|
||||
unaryEncodeDiffPrivacy(500, numValues, 0.75, 0.25).length,
|
||||
4,
|
||||
"Basic dp runs with unexpected input"
|
||||
);
|
||||
Assert.equal(
|
||||
unaryEncodeDiffPrivacy(-100, numValues, 0.75, 0.25).length,
|
||||
4,
|
||||
"Basic dp runs with unexpected input"
|
||||
);
|
||||
Assert.equal(
|
||||
unaryEncodeDiffPrivacy(1, numValues, 0.75, 0.25).length,
|
||||
4,
|
||||
"Basic dp runs with typical values"
|
||||
);
|
||||
Assert.equal(
|
||||
unaryEncodeDiffPrivacy(1, numValues, 0.8, 0.6).length,
|
||||
4,
|
||||
"Basic dp runs with typical values"
|
||||
);
|
||||
});
|
||||
|
||||
add_task(function test_differentialPrivacy() {
|
||||
const modelData = { ...jsonModelData, rescale: true };
|
||||
const model = FeatureModel.fromJSON(modelData);
|
||||
const result = model.computeInterestVector({
|
||||
dataForIntervals: SQL_RESULT_DATA,
|
||||
indexSchema: SCHEMA,
|
||||
applyThresholding: true,
|
||||
applyDifferentialPrivacy: true,
|
||||
});
|
||||
Assert.equal(
|
||||
result.parenting,
|
||||
"001",
|
||||
"Threshold is applied with differential privacy"
|
||||
);
|
||||
Assert.equal(
|
||||
result[SPECIAL_FEATURE_CLICK],
|
||||
2,
|
||||
"Should include rescaled raw click with no dp"
|
||||
);
|
||||
});
|
||||
@@ -15,6 +15,8 @@ support-files = ["topstories.json"]
|
||||
|
||||
["test_HighlightsFeed.js"]
|
||||
|
||||
["test_InferredFeatureModel.js"]
|
||||
|
||||
["test_NewTabGleanUtils.js"]
|
||||
|
||||
["test_NewTabMessaging.js"]
|
||||
|
||||
Reference in New Issue
Block a user