tubestation/browser/extensions/activity-stream/lib/TopStoriesFeed.jsm

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";

const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://gre/modules/Services.jsm");
Cu.import("resource://gre/modules/NewTabUtils.jsm");
Cu.importGlobalProperties(["fetch"]);

const {actionTypes: at, actionCreators: ac} = Cu.import("resource://activity-stream/common/Actions.jsm", {});
const {Prefs} = Cu.import("resource://activity-stream/lib/ActivityStreamPrefs.jsm", {});
const {shortURL} = Cu.import("resource://activity-stream/lib/ShortURL.jsm", {});
const {SectionsManager} = Cu.import("resource://activity-stream/lib/SectionsManager.jsm", {});
const {UserDomainAffinityProvider} = Cu.import("resource://activity-stream/lib/UserDomainAffinityProvider.jsm", {});
const {PersistentCache} = Cu.import("resource://activity-stream/lib/PersistentCache.jsm", {});

XPCOMUtils.defineLazyModuleGetter(this, "perfService", "resource://activity-stream/common/PerfService.jsm");

const STORIES_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes
const TOPICS_UPDATE_TIME = 3 * 60 * 60 * 1000; // 3 hours
const DOMAIN_AFFINITY_UPDATE_TIME = 24 * 60 * 60 * 1000; // 24 hours
const STORIES_NOW_THRESHOLD = 24 * 60 * 60 * 1000; // 24 hours
const SECTION_ID = "topstories";

this.TopStoriesFeed = class TopStoriesFeed {
  constructor() {
    this.spocsPerNewTabs = 0;
    this.newTabsSinceSpoc = 0;
    this.contentUpdateQueue = [];
    this.cache = new PersistentCache(SECTION_ID, true);
  }

  init() {
    const initFeed = () => {
      SectionsManager.enableSection(SECTION_ID);
      try {
        const options = SectionsManager.sections.get(SECTION_ID).options;
        const apiKey = this.getApiKeyFromPref(options.api_key_pref);
        this.stories_endpoint = this.produceFinalEndpointUrl(options.stories_endpoint, apiKey);
        this.topics_endpoint = this.produceFinalEndpointUrl(options.topics_endpoint, apiKey);
        this.read_more_endpoint = options.read_more_endpoint;
        this.stories_referrer = options.stories_referrer;
        this.personalized = options.personalized;
        this.show_spocs = options.show_spocs;
        this.maxHistoryQueryResults = options.maxHistoryQueryResults;
        this.storiesLastUpdated = 0;
        this.topicsLastUpdated = 0;
        this.affinityLastUpdated = 0;

        this.loadCachedData();
        this.fetchStories();
        this.fetchTopics();
      } catch (e) {
        Cu.reportError(`Problem initializing top stories feed: ${e.message}`);
      }
    };
    SectionsManager.onceInitialized(initFeed);
  }

  uninit() {
    SectionsManager.disableSection(SECTION_ID);
  }

  async fetchStories() {
    if (!this.stories_endpoint) {
      return;
    }
    try {
      const response = await fetch(this.stories_endpoint);
      if (!response.ok) {
        throw new Error(`Stories endpoint returned unexpected status: ${response.status}`);
      }

      const body = await response.json();
      this.updateSettings(body.settings);
      this.stories = this.rotate(this.transform(body.recommendations));
      this.spocs = this.show_spocs && this.transform(body.spocs).filter(s => s.score >= s.min_score);

      this.dispatchUpdateEvent(this.storiesLastUpdated, {rows: this.stories});
      body._timestamp = this.storiesLastUpdated = Date.now();
      // This is filtered so an update function can return true to retry on the next run
      this.contentUpdateQueue = this.contentUpdateQueue.filter(update => update());

      this.cache.set("stories", body);
    } catch (error) {
      Cu.reportError(`Failed to fetch content: ${error.message}`);
    }
  }

  async loadCachedData() {
    const data = await this.cache.get();
    let stories = data.stories && data.stories.recommendations;
    let topics = data.topics && data.topics.topics;
    if (stories && stories.length > 0 && this.storiesLastUpdated === 0) {
      this.updateSettings(data.stories.settings);
      const rows = this.transform(stories);
      this.dispatchUpdateEvent(this.storiesLastUpdated, {rows});
      this.storiesLastUpdated = data.stories._timestamp;
    }
    if (topics && topics.length > 0 && this.topicsLastUpdated === 0) {
      this.dispatchUpdateEvent(this.topicsLastUpdated, {topics, read_more_endpoint: this.read_more_endpoint});
      this.topicsLastUpdated = data.topics._timestamp;
    }
  }

  transform(items) {
    if (!items) {
      return [];
    }

    return items
      .filter(s => !NewTabUtils.blockedLinks.isBlocked({"url": s.url}))
      .map(s => ({
        "guid": s.id,
        "hostname": shortURL(Object.assign({}, s, {url: s.url})),
        "type": (Date.now() - (s.published_timestamp * 1000)) <= STORIES_NOW_THRESHOLD ? "now" : "trending",
        "context": s.context,
        "icon": s.icon,
        "title": s.title,
        "description": s.excerpt,
        "image": this.normalizeUrl(s.image_src),
        "referrer": this.stories_referrer,
        "url": s.url,
        "min_score": s.min_score || 0,
        "score": this.personalized ? this.affinityProvider.calculateItemRelevanceScore(s) : 1
      }))
      .sort(this.personalized ? this.compareScore : (a, b) => 0);
  }

  async fetchTopics() {
    if (!this.topics_endpoint) {
      return;
    }
    try {
      const response = await fetch(this.topics_endpoint);
      if (!response.ok) {
        throw new Error(`Topics endpoint returned unexpected status: ${response.status}`);
      }
      const body = await response.json();
      const {topics} = body;
      if (topics) {
        this.dispatchUpdateEvent(this.topicsLastUpdated, {topics, read_more_endpoint: this.read_more_endpoint});
        body._timestamp = this.topicsLastUpdated = Date.now();
        this.cache.set("topics", body);
      }
    } catch (error) {
      Cu.reportError(`Failed to fetch topics: ${error.message}`);
    }
  }

  dispatchUpdateEvent(lastUpdated, data) {
    SectionsManager.updateSection(SECTION_ID, data, lastUpdated === 0);
  }

  compareScore(a, b) {
    return b.score - a.score;
  }

  updateSettings(settings) {
    if (!this.personalized) {
      return;
    }

    this.spocsPerNewTabs = settings.spocsPerNewTabs;

    if (!this.affinityProvider || (Date.now() - this.affinityLastUpdated >= DOMAIN_AFFINITY_UPDATE_TIME)) {
      const start = perfService.absNow();
      this.affinityProvider = new UserDomainAffinityProvider(
        settings.timeSegments,
        settings.domainAffinityParameterSets,
        this.maxHistoryQueryResults);

      this.store.dispatch(ac.PerfEvent({
        event: "topstories.domain.affinity.calculation.ms",
        value: Math.round(perfService.absNow() - start)
      }));
      this.affinityLastUpdated = Date.now();
    }
  }

  // If personalization is turned on we have to rotate stories on the client.
  // An item can only be on top for two iterations (1hr) before it gets moved
  // to the end. This will later be improved based on interactions/impressions.
  rotate(items) {
    if (!this.personalized || items.length <= 3) {
      return items;
    }

    if (!this.topItems) {
      this.topItems = new Map();
    }

    // This avoids an infinite recursion if for some reason the feed stops
    // changing. Otherwise, there's a chance we'd be rotating forever to
    // find an item we haven't displayed on top yet.
    if (this.topItems.size >= items.length) {
      this.topItems.clear();
    }

    const guid = items[0].guid;
    if (!this.topItems.has(guid)) {
      this.topItems.set(guid, 0);
    } else {
      const val = this.topItems.get(guid) + 1;
      this.topItems.set(guid, val);
      if (val >= 2) {
        items.push(items.shift());
        this.rotate(items);
      }
    }
    return items;
  }

  getApiKeyFromPref(apiKeyPref) {
    if (!apiKeyPref) {
      return apiKeyPref;
    }

    return new Prefs().get(apiKeyPref) || Services.prefs.getCharPref(apiKeyPref);
  }

  produceFinalEndpointUrl(url, apiKey) {
    if (!url) {
      return url;
    }
    if (url.includes("$apiKey") && !apiKey) {
      throw new Error(`An API key was specified but none configured: ${url}`);
    }
    return url.replace("$apiKey", apiKey);
  }

  // Need to remove parenthesis from image URLs as React will otherwise
  // fail to render them properly as part of the card template.
  normalizeUrl(url) {
    if (url) {
      return url.replace(/\(/g, "%28").replace(/\)/g, "%29");
    }
    return url;
  }

  maybeAddSpoc(target) {
    if (!this.show_spocs || !this.store.getState().Prefs.values.showSponsored) {
      return;
    }

    if (this.newTabsSinceSpoc === 0 || this.newTabsSinceSpoc === this.spocsPerNewTabs) {
      const updateContent = () => {
        if (!this.spocs || !this.spocs.length) {
          // We have stories but no spocs so there's nothing to do and this update can be
          // removed from the queue.
          return false;
        }

        // Create a new array with a spoc inserted at index 2
        // For now we're using the top scored spoc until we can support viewability based rotation
        const position = SectionsManager.sections.get(SECTION_ID).order;
        let rows = this.store.getState().Sections[position].rows.slice(0, this.stories.length);
        rows.splice(2, 0, this.spocs[0]);

        // Send a content update to the target tab
        const action = {type: at.SECTION_UPDATE, meta: {skipMain: true}, data: Object.assign({rows}, {id: SECTION_ID})};
        this.store.dispatch(ac.SendToContent(action, target));
        return false;
      };

      if (this.stories) {
        updateContent();
      } else {
        // Delay updating tab content until initial data has been fetched
        this.contentUpdateQueue.push(updateContent);
      }

      this.newTabsSinceSpoc = 0;
    }
    this.newTabsSinceSpoc++;
  }

  onAction(action) {
    switch (action.type) {
      case at.INIT:
        this.init();
        break;
      case at.SYSTEM_TICK:
        if (Date.now() - this.storiesLastUpdated >= STORIES_UPDATE_TIME) {
          this.fetchStories();
        }
        if (Date.now() - this.topicsLastUpdated >= TOPICS_UPDATE_TIME) {
          this.fetchTopics();
        }
        break;
      case at.UNINIT:
        this.uninit();
        break;
      case at.NEW_TAB_REHYDRATED:
        this.maybeAddSpoc(action.meta.fromTarget);
        break;
      case at.SECTION_OPTIONS_CHANGED:
        if (action.data === SECTION_ID) {
          this.uninit();
          this.init();
        }
        break;
      case at.PLACES_LINK_BLOCKED:
        if (this.spocs) {
          this.spocs = this.spocs.filter(s => s.url !== action.data.url);
        }
        break;
    }
  }
};

this.STORIES_UPDATE_TIME = STORIES_UPDATE_TIME;
this.TOPICS_UPDATE_TIME = TOPICS_UPDATE_TIME;
this.SECTION_ID = SECTION_ID;
this.EXPORTED_SYMBOLS = ["TopStoriesFeed", "STORIES_UPDATE_TIME", "TOPICS_UPDATE_TIME", "DOMAIN_AFFINITY_UPDATE_TIME", "SECTION_ID"];