Bug 895476: Integrate application reputation with download manager, disable remote lookups for application reputation (r=paolo)

browser/components/downloads/src/DownloadsCommon.jsm

@@ -840,6 +840,9 @@ DownloadsDataItem.prototype = {
     } else if (this._download.error &&
                this._download.error.becauseBlockedByParentalControls) {
       this.state = nsIDM.DOWNLOAD_BLOCKED_PARENTAL;
+    } else if (this._download.error &&
+               this._download.error.becauseBlockedByReputationCheck) {
+      this.state = nsIDM.DOWNLOAD_DIRTY;
     } else if (this._download.error) {
       this.state = nsIDM.DOWNLOAD_FAILED;
     } else if (this._download.canceled && this._download.hasPartialData) {

toolkit/components/downloads/ApplicationReputation.cpp

@@ -128,11 +128,16 @@ PendingLookup::HandleEvent(const nsACString& tables) {
     return OnComplete(true, NS_OK);
   }
 
+#if 0
   nsresult rv = SendRemoteQuery();
   if (NS_FAILED(rv)) {
     return OnComplete(false, rv);
   }
   return NS_OK;
+#else
+  // Revert when remote lookups are enabled (bug 933432)
+  return OnComplete(false, NS_OK);
+#endif
 }
 
 nsresult

toolkit/components/downloads/test/unit/test_app_rep.js

@@ -103,6 +103,8 @@ function run_test() {
   run_next_test();
 }
 
+/*
+// Uncomment when remote lookups are enabled (bug 933432)
 add_test(function test_shouldBlock() {
   gAppRep.queryReputation({
     sourceURI: createURI("http://evil.com"),
@@ -124,6 +126,7 @@ add_test(function test_shouldNotBlock() {
     run_next_test();
   });
 });
+*/
 
 add_test(function test_nullSourceURI() {
   gAppRep.queryReputation({
@@ -161,6 +164,8 @@ add_test(function test_disabled() {
   });
 });
 
+/*
+// Uncomment when remote lookups are enabled (bug 933432)
 add_test(function test_garbage() {
   Services.prefs.setCharPref("browser.safebrowsing.appRepURL",
                              "http://localhost:4444/download");
@@ -174,6 +179,7 @@ add_test(function test_garbage() {
     run_next_test();
   });
 });
+*/
 
 // Set up the local whitelist.
 add_test(function test_local_list() {

toolkit/components/jsdownloads/src/DownloadCore.jsm

@@ -417,7 +417,18 @@ Download.prototype = {
         // Execute the actual download through the saver object.
         yield this.saver.execute(DS_setProgressBytes.bind(this),
                                  DS_setProperties.bind(this));
-
+        // Check for application reputation, which requires the entire file to
+        // be downloaded.
+        if (yield DownloadIntegration.shouldBlockForReputationCheck(this)) {
+          // Delete the target file that BackgroundFileSaver already moved
+          // into place.
+          try {
+            yield OS.File.remove(this.target.path);
+          } catch (ex) {
+            Cu.reportError(ex);
+          }
+          throw new DownloadError({ becauseBlockedByReputationCheck: true });
+        }
         // Update the status properties for a successful download.
         this.progress = 100;
         this.succeeded = true;
@@ -456,6 +467,7 @@ Download.prototype = {
           this.speed = 0;
           this._notifyChange();
           if (this.succeeded) {
+            dump("we're good\n");
             yield DownloadIntegration.downloadDone(this);
 
             this._deferSucceeded.resolve();
@@ -1185,7 +1197,8 @@ function DownloadError(aProperties)
   if (aProperties.message) {
     this.message = aProperties.message;
   } else if (aProperties.becauseBlocked ||
-             aProperties.becauseBlockedByParentalControls) {
+             aProperties.becauseBlockedByParentalControls ||
+             aProperties.becauseBlockedByReputationCheck) {
     this.message = "Download blocked.";
   } else {
     let exception = new Components.Exception("", this.result);
@@ -1209,8 +1222,10 @@ function DownloadError(aProperties)
   if (aProperties.becauseBlockedByParentalControls) {
     this.becauseBlocked = true;
     this.becauseBlockedByParentalControls = true;
-  }
-  else if (aProperties.becauseBlocked) {
+  } else if (aProperties.becauseBlockedByReputationCheck) {
+    this.becauseBlocked = true;
+    this.becauseBlockedByReputationCheck = true;
+  } else if (aProperties.becauseBlocked) {
     this.becauseBlocked = true;
   }
 
@@ -1246,6 +1261,12 @@ DownloadError.prototype = {
    * disallowed by the Parental Controls or Family Safety features on Windows.
    */
   becauseBlockedByParentalControls: false,
+
+  /**
+   * Indicates the download was blocked because it failed the reputation check
+   * and may be malware.
+   */
+  becauseBlockedByReputationCheck: false,
 };
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -1374,7 +1395,15 @@ DownloadSaver.prototype = {
   {
     throw new Error("Not implemented.");
   },
-};
+
+  /**
+   * Returns the SHA-256 hash of the downloaded file, if it exists.
+   */
+  getSha256Hash: function ()
+  {
+    throw new Error("Not implemented.");
+  }
+}; // DownloadSaver
 
 /**
  * Creates a new DownloadSaver object from its serializable representation.
@@ -1426,6 +1455,12 @@ DownloadCopySaver.prototype = {
    */
   _canceled: false,
 
+  /**
+   * Save the SHA-256 hash in raw bytes of the downloaded file. This is null
+   * unless BackgroundFileSaver has successfully completed saving the file.
+   */
+  _sha256Hash: null,
+
   /**
    * True if the associated download has already been added to browsing history.
    */
@@ -1497,14 +1532,11 @@ DownloadCopySaver.prototype = {
           // returned by this download execution function.
           backgroundFileSaver.observer = {
             onTargetChange: function () { },
-            onSaveComplete: function DCSE_onSaveComplete(aSaver, aStatus)
-            {
-              // Free the reference cycle, to release resources earlier.
-              backgroundFileSaver.observer = null;
-              this._backgroundFileSaver = null;
-
+            onSaveComplete: (aSaver, aStatus) => {
               // Send notifications now that we can restart if needed.
               if (Components.isSuccessCode(aStatus)) {
+                // Save the hash before freeing backgroundFileSaver.
+                this._sha256Hash = aSaver.sha256Hash;
                 deferSaveComplete.resolve();
               } else {
                 // Infer the origin of the error from the failure code, because
@@ -1512,6 +1544,9 @@ DownloadCopySaver.prototype = {
                 let properties = { result: aStatus, inferCause: true };
                 deferSaveComplete.reject(new DownloadError(properties));
               }
+              // Free the reference cycle, to release resources earlier.
+              backgroundFileSaver.observer = null;
+              this._backgroundFileSaver = null;
             },
           };
 
@@ -1609,6 +1644,8 @@ DownloadCopySaver.prototype = {
                 }
               }
 
+              // Enable hashing before setting the target.
+              backgroundFileSaver.enableSha256();
               if (partFilePath) {
                 // If we actually resumed a request, append to the partial data.
                 if (resumeAttempted) {
@@ -1727,6 +1764,14 @@ DownloadCopySaver.prototype = {
     serializeUnknownProperties(this, serializable);
     return serializable;
   },
+
+  /**
+   * Implements "DownloadSaver.getSha256Hash"
+   */
+  getSha256Hash: function ()
+  {
+    return this._sha256Hash;
+  }
 };
 
 /**
@@ -1767,6 +1812,13 @@ function DownloadLegacySaver()
 DownloadLegacySaver.prototype = {
   __proto__: DownloadSaver.prototype,
 
+  /**
+   * Save the SHA-256 hash in raw bytes of the downloaded file. This may be
+   * null when nsExternalHelperAppService (and thus BackgroundFileSaver) is not
+   * invoked.
+   */
+  _sha256Hash: null,
+
   /**
    * nsIRequest object associated to the status and progress updates we
    * received.  This object is null before we receive the first status and
@@ -2018,6 +2070,25 @@ DownloadLegacySaver.prototype = {
     // DownloadCopySaver for the purpose of serialization.
     return DownloadCopySaver.prototype.toSerializable.call(this);
   },
+
+  /**
+   * Implements "DownloadSaver.getSha256Hash".
+   */
+  getSha256Hash: function ()
+  {
+    if (this.copySaver) {
+      return this.copySaver.getSha256Hash();
+    }
+    return this._sha256Hash;
+  },
+
+  /**
+   * Called by the nsITransfer implementation when the hash is available.
+   */
+  setSha256Hash: function (hash)
+  {
+    this._sha256Hash = hash;
+  },
 };
 
 /**

toolkit/components/jsdownloads/src/DownloadIntegration.jsm

@@ -71,6 +71,10 @@ XPCOMUtils.defineLazyGetter(this, "gParentalControlsService", function() {
   return null;
 });
 
+XPCOMUtils.defineLazyServiceGetter(this, "gApplicationReputationService",
+           "@mozilla.org/downloads/application-reputation-service;1",
+           Ci.nsIApplicationReputationService);
+
 /**
  * ArrayBufferView representing the bytes to be written to the "Zone.Identifier"
  * Alternate Data Stream to mark a file as coming from the Internet zone.
@@ -125,6 +129,8 @@ this.DownloadIntegration = {
   dontLoadObservers: false,
   dontCheckParentalControls: false,
   shouldBlockInTest: false,
+  dontCheckApplicationReputation: false,
+  shouldBlockInTestForApplicationReputation: false,
   dontOpenFileAndFolder: false,
   downloadDoneCalled: false,
   _deferTestOpenFile: null,
@@ -401,6 +407,41 @@ this.DownloadIntegration = {
     return Promise.resolve(shouldBlock);
   },
 
+  /**
+   * Checks to determine whether to block downloads because they might be
+   * malware, based on application reputation checks.
+   *
+   * aParam aDownload
+   *        The download object.
+   *
+   * @return {Promise}
+   * @resolves The boolean indicates to block downloads or not.
+   */
+  shouldBlockForReputationCheck: function (aDownload) {
+    if (this.dontCheckApplicationReputation) {
+      return Promise.resolve(this.shouldBlockInTestForApplicationReputation);
+    }
+    let hash;
+    try {
+      hash = aDownload.saver.getSha256Hash();
+    } catch (ex) {
+      // Bail if DownloadSaver doesn't have a hash.
+      return Promise.resolve(false);
+    }
+    if (!hash) {
+      return Promise.resolve(false);
+    }
+    let deferred = Promise.defer();
+    gApplicationReputationService.queryReputation({
+      sourceURI: NetUtil.newURI(aDownload.source.url),
+      fileSize: aDownload.currentBytes,
+      sha256Hash: hash },
+      function onComplete(aShouldBlock, aRv) {
+        deferred.resolve(aShouldBlock);
+      });
+    return deferred.promise;
+  },
+
   /**
    * Performs platform-specific operations when a download is done.
    *

toolkit/components/jsdownloads/src/DownloadLegacy.js

@@ -117,6 +117,11 @@ DownloadLegacyTransfer.prototype = {
       // The last file has been received, or the download failed.  Wait for the
       // associated Download object to be available before notifying.
       this._deferDownload.promise.then(download => {
+	// At this point, the hash has been set and we need to copy it to the
+	// DownloadSaver.
+	if (Components.isSuccessCode(aStatus)) {
+          download.saver.setSha256Hash(this._sha256Hash);
+        }
         download.saver.onTransferFinished(aRequest, aStatus);
       }).then(null, Cu.reportError);
 
@@ -231,7 +236,10 @@ DownloadLegacyTransfer.prototype = {
     }.bind(this)).then(null, Cu.reportError);
   },
 
-  setSha256Hash: function () { },
+  setSha256Hash: function (hash)
+  {
+    this._sha256Hash = hash;
+  },
 
   //////////////////////////////////////////////////////////////////////////////
   //// Private methods and properties
@@ -254,6 +262,11 @@ DownloadLegacyTransfer.prototype = {
    * that cancel the download.
    */
   _componentFailed: false,
+
+  /**
+   * Save the SHA-256 hash in raw bytes of the downloaded file.
+   */
+  _sha256Hash: null,
 };
 
 ////////////////////////////////////////////////////////////////////////////////

toolkit/components/jsdownloads/test/unit/common_test_Download.js

@@ -164,6 +164,7 @@ add_task(function test_basic_tryToKeepPartialData()
   yield promiseVerifyContents(download.target.path,
                               TEST_DATA_SHORT + TEST_DATA_SHORT);
   do_check_false(yield OS.File.exists(download.target.partFilePath));
+  do_check_eq(32, download.saver.getSha256Hash().length);
 });
 
 /**
@@ -350,6 +351,7 @@ add_task(function test_empty_progress_tryToKeepPartialData()
   // The target file should now have been created, and the ".part" file deleted.
   do_check_eq((yield OS.File.stat(download.target.path)).size, 0);
   do_check_false(yield OS.File.exists(download.target.partFilePath));
+  do_check_eq(32, download.saver.getSha256Hash().length);
 });
 
 /**
@@ -1414,6 +1416,59 @@ add_task(function test_blocked_parental_controls_httpstatus450()
   do_check_false(yield OS.File.exists(download.target.path));
 });
 
+/**
+ * Check that DownloadCopySaver can always retrieve the hash.
+ * DownloadLegacySaver can only retrieve the hash when
+ * nsIExternalHelperAppService is invoked.
+ */
+add_task(function test_getSha256Hash()
+{
+  if (!gUseLegacySaver) {
+    let download = yield promiseStartDownload(httpUrl("source.txt"));
+    yield promiseDownloadStopped(download);
+    do_check_true(download.stopped);
+    do_check_eq(32, download.saver.getSha256Hash().length);
+  }
+});
+
+/**
+ * Checks that application reputation blocks the download and the target file
+ * does not exist.
+ */
+add_task(function test_blocked_applicationReputation()
+{
+  function cleanup() {
+    DownloadIntegration.shouldBlockInTestForApplicationReputation = false;
+  }
+  do_register_cleanup(cleanup);
+  DownloadIntegration.shouldBlockInTestForApplicationReputation = true;
+
+  let download;
+  try {
+    if (!gUseLegacySaver) {
+      // When testing DownloadCopySaver, we want to check that the promise
+      // returned by the "start" method is rejected.
+      download = yield promiseNewDownload();
+      yield download.start();
+    } else {
+      // When testing DownloadLegacySaver, we cannot be sure whether we are
+      // testing the promise returned by the "start" method or we are testing
+      // the "error" property checked by promiseDownloadStopped.  This happens
+      // because we don't have control over when the download is started.
+      download = yield promiseStartLegacyDownload();
+      yield promiseDownloadStopped(download);
+    }
+    do_throw("The download should have blocked.");
+  } catch (ex if ex instanceof Downloads.Error && ex.becauseBlocked) {
+    do_check_true(ex.becauseBlockedByReputationCheck);
+    do_check_true(download.error.becauseBlockedByReputationCheck);
+  }
+
+  // Now that the download is blocked, the target file should not exist.
+  do_check_false(yield OS.File.exists(download.target.path));
+  cleanup();
+});
+
 /**
  * download.showContainingDirectory() action
  */
@@ -1770,4 +1825,3 @@ add_task(function test_launchWhenSucceeded_deleteTempFileOnExit() {
   do_check_false(yield OS.File.exists(autoDeleteTargetPathTwo));
   do_check_true(yield OS.File.exists(noAutoDeleteTargetPath));
 });
-

toolkit/components/jsdownloads/test/unit/head.js

@@ -810,6 +810,8 @@ add_task(function test_common_initialize()
   DownloadIntegration.dontLoadObservers = true;
   // Disable the parental controls checking.
   DownloadIntegration.dontCheckParentalControls = true;
+  // Disable application reputation checks.
+  DownloadIntegration.dontCheckApplicationReputation = true;
   // Disable the calls to the OS to launch files and open containing folders
   DownloadIntegration.dontOpenFileAndFolder = true;
   DownloadIntegration._deferTestOpenFile = Promise.defer();