Bug 1715136 - Make SkConvolutionFilter1D::AddFilter fallible. r=aosmond

This allows SkConvolutionFilter1D::AddFilter to simply return false on
an OOM rather than cause an unrecoverable error.

Differential Revision: https://phabricator.services.mozilla.com/D204867

gfx/2d/SkConvolver.cpp

@@ -5,7 +5,6 @@
 // found in the gfx/skia/LICENSE file.
 
 #include "SkConvolver.h"
-#include "mozilla/Vector.h"
 
 #ifdef USE_SSE2
 #  include "mozilla/SSE.h"
@@ -235,8 +234,10 @@ class CircularRowBuffer {
       : fRowByteWidth(destRowPixelWidth * 4),
         fNumRows(maxYFilterSize),
         fNextRow(0),
-        fNextRowCoordinate(firstInputRow) {
-    fBuffer.resize(fRowByteWidth * maxYFilterSize);
+        fNextRowCoordinate(firstInputRow) {}
+
+  bool AllocBuffer() {
+    return fBuffer.resize(fRowByteWidth * fNumRows) &&
            fRowAddresses.resize(fNumRows);
   }
 
@@ -288,7 +289,7 @@ class CircularRowBuffer {
 
  private:
   // The buffer storing the rows. They are packed, each one fRowByteWidth.
-  std::vector<unsigned char> fBuffer;
+  mozilla::Vector<unsigned char> fBuffer;
 
   // Number of bytes per row in the |buffer|.
   int fRowByteWidth;
@@ -305,14 +306,14 @@ class CircularRowBuffer {
   int fNextRowCoordinate;
 
   // Buffer used by GetRowAddresses().
-  std::vector<unsigned char*> fRowAddresses;
+  mozilla::Vector<unsigned char*> fRowAddresses;
 };
 
 SkConvolutionFilter1D::SkConvolutionFilter1D() : fMaxFilter(0) {}
 
 SkConvolutionFilter1D::~SkConvolutionFilter1D() = default;
 
-void SkConvolutionFilter1D::AddFilter(int filterOffset,
+bool SkConvolutionFilter1D::AddFilter(int filterOffset,
                                       const ConvolutionFixed* filterValues,
                                       int filterLength) {
   // It is common for leading/trailing filter values to be zeros. In such
@@ -336,8 +337,9 @@ void SkConvolutionFilter1D::AddFilter(int filterOffset,
     filterLength = lastNonZero + 1 - firstNonZero;
     MOZ_ASSERT(filterLength > 0);
 
-    fFilterValues.insert(fFilterValues.end(), &filterValues[firstNonZero],
-                         &filterValues[lastNonZero + 1]);
+    if (!fFilterValues.append(&filterValues[firstNonZero], filterLength)) {
+      return false;
+    }
   } else {
     // Here all the factors were zeroes.
     filterLength = 0;
@@ -345,11 +347,17 @@ void SkConvolutionFilter1D::AddFilter(int filterOffset,
 
   FilterInstance instance = {
       // We pushed filterLength elements onto fFilterValues
-      int(fFilterValues.size()) - filterLength, filterOffset, filterLength,
+      int(fFilterValues.length()) - filterLength, filterOffset, filterLength,
       filterSize};
-  fFilters.push_back(instance);
+  if (!fFilters.append(instance)) {
+    if (filterLength > 0) {
+      fFilterValues.shrinkBy(filterLength);
+    }
+    return false;
+  }
 
   fMaxFilter = std::max(fMaxFilter, filterLength);
+  return true;
 }
 
 bool SkConvolutionFilter1D::ComputeFilterValues(
@@ -383,10 +391,13 @@ bool SkConvolutionFilter1D::ComputeFilterValues(
 
   int32_t filterValueCount = int32_t(ceilf(aDstSize * srcSupport * 2));
   if (aDstSize > maxToPassToReserveAdditional || filterValueCount < 0 ||
-      filterValueCount > maxToPassToReserveAdditional) {
+      filterValueCount > maxToPassToReserveAdditional ||
+      !reserveAdditional(aDstSize, filterValueCount)) {
     return false;
   }
-  reserveAdditional(aDstSize, filterValueCount);
+  size_t oldFiltersLength = fFilters.length();
+  size_t oldFilterValuesLength = fFilterValues.length();
+  int oldMaxFilter = fMaxFilter;
   for (int32_t destI = 0; destI < aDstSize; destI++) {
     // This is the pixel in the source directly under the pixel in the dest.
     // Note that we base computations on the "center" of the pixels. To see
@@ -443,7 +454,12 @@ bool SkConvolutionFilter1D::ComputeFilterValues(
     ConvolutionFixed leftovers = ToFixed(1) - fixedSum;
     fixedFilterValues[filterCount / 2] += leftovers;
 
-    AddFilter(int32_t(srcBegin), fixedFilterValues.begin(), filterCount);
+    if (!AddFilter(int32_t(srcBegin), fixedFilterValues.begin(), filterCount)) {
+      fFilters.shrinkTo(oldFiltersLength);
+      fFilterValues.shrinkTo(oldFilterValuesLength);
+      fMaxFilter = oldMaxFilter;
+      return false;
+    }
   }
 
   return maxFilter() > 0 && numValues() == aDstSize;
@@ -515,6 +531,9 @@ bool BGRAConvolve2D(const unsigned char* sourceData, int sourceByteRowStride,
   }
 
   CircularRowBuffer rowBuffer(rowBufferWidth, rowBufferHeight, filterOffset);
+  if (!rowBuffer.AllocBuffer()) {
+    return false;
+  }
 
   // Loop over every possible output row, processing just enough horizontal
   // convolutions to run each subsequent vertical convolution.

gfx/2d/SkConvolver.h

@@ -10,7 +10,7 @@
 #include "mozilla/Assertions.h"
 #include <cfloat>
 #include <cmath>
-#include <vector>
+#include "mozilla/Vector.h"
 
 namespace skia {
 
@@ -81,11 +81,11 @@ class SkConvolutionFilter1D {
 
   // Returns the number of filters in this filter. This is the dimension of the
   // output image.
-  int numValues() const { return static_cast<int>(fFilters.size()); }
+  int numValues() const { return static_cast<int>(fFilters.length()); }
 
-  void reserveAdditional(int filterCount, int filterValueCount) {
-    fFilters.reserve(fFilters.size() + filterCount);
-    fFilterValues.reserve(fFilterValues.size() + filterValueCount);
+  bool reserveAdditional(int filterCount, int filterValueCount) {
+    return fFilters.reserve(fFilters.length() + filterCount) &&
+           fFilterValues.reserve(fFilterValues.length() + filterValueCount);
   }
 
   // Appends the given list of scaling values for generating a given output
@@ -98,7 +98,7 @@ class SkConvolutionFilter1D {
   // brighness of the image.
   //
   // The filterLength must be > 0.
-  void AddFilter(int filterOffset, const ConvolutionFixed* filterValues,
+  bool AddFilter(int filterOffset, const ConvolutionFixed* filterValues,
                  int filterLength);
 
   // Retrieves a filter for the given |valueOffset|, a position in the output
@@ -139,12 +139,12 @@ class SkConvolutionFilter1D {
   };
 
   // Stores the information for each filter added to this class.
-  std::vector<FilterInstance> fFilters;
+  mozilla::Vector<FilterInstance> fFilters;
 
   // We store all the filter values in this flat list, indexed by
   // |FilterInstance.data_location| to avoid the mallocs required for storing
   // each one separately.
-  std::vector<ConvolutionFixed> fFilterValues;
+  mozilla::Vector<ConvolutionFixed> fFilterValues;
 
   // The maximum size of any filter we've added.
   int fMaxFilter;