Bug 1933572 - Be more stringent about which SurfaceDescriptors we serialize for moz2d recording IPC. r=tnikkel a=pascalc

Original Revision: https://phabricator.services.mozilla.com/D244105

Differential Revision: https://phabricator.services.mozilla.com/D254904

dom/canvas/CanvasRenderingContext2D.cpp

@@ -5501,6 +5501,52 @@ static Matrix ComputeRotationMatrix(gfxFloat aRotatedWidth,
       .PostTranslate(shiftLeftTopToOrigin);
 }
 
+// -
+
+Maybe<layers::SurfaceDescriptor> ValidSurfaceDescriptorForRemoteCanvas2d(
+    const layers::SurfaceDescriptor& sdConst) {
+  auto sd = sdConst;  // Copy, so we can mutate it.
+  if (sd.type() != layers::SurfaceDescriptor::TSurfaceDescriptorGPUVideo) {
+    return Nothing();
+  }
+
+  auto& sdv = sd.get_SurfaceDescriptorGPUVideo();
+  if (sdv.type() !=
+      layers::SurfaceDescriptorGPUVideo::TSurfaceDescriptorRemoteDecoder) {
+    return Nothing();
+  }
+  auto& sdrd = sdv.get_SurfaceDescriptorRemoteDecoder();
+  auto& subdesc = sdrd.subdesc();
+  switch (subdesc.type()) {
+    case layers::RemoteDecoderVideoSubDescriptor::Tnull_t:
+      break;
+#ifdef XP_MACOSX
+    case layers::RemoteDecoderVideoSubDescriptor::
+        TSurfaceDescriptorMacIOSurface: {
+      const auto& ssd = subdesc.get_SurfaceDescriptorMacIOSurface();
+      if (ssd.gpuFence()) {
+        return Nothing();
+      }
+      break;
+    }
+#endif
+#ifdef XP_WIN
+    case layers::RemoteDecoderVideoSubDescriptor::TSurfaceDescriptorD3D10: {
+      if (!StaticPrefs::gfx_canvas_remote_use_draw_image_fast_path_d3d()) {
+        return Nothing();
+      }
+      auto& ssd = subdesc.get_SurfaceDescriptorD3D10();
+      ssd.handle() =
+          nullptr;  // Not IPC-able, but it's just an optimization to have this.
+      break;
+    }
+#endif
+    default:
+      return Nothing();
+  }
+  return Some(sd);
+}
+
 static Maybe<layers::SurfaceDescriptor>
 MaybeGetSurfaceDescriptorForRemoteCanvas(
     const SurfaceFromElementResult& aResult) {
@@ -5512,43 +5558,9 @@ MaybeGetSurfaceDescriptorForRemoteCanvas(
     return Nothing();
   }
 
-  Maybe<layers::SurfaceDescriptor> sd;
+  const auto sd = aResult.mLayersImage->GetDesc();
-  sd = aResult.mLayersImage->GetDesc();
+  if (!sd) return Nothing();
-  if (sd.isNothing() ||
+  return ValidSurfaceDescriptorForRemoteCanvas2d(*sd);
-      sd.ref().type() !=
-          layers::SurfaceDescriptor::TSurfaceDescriptorGPUVideo) {
-    return Nothing();
-  }
-
-  auto& sdv = sd.ref().get_SurfaceDescriptorGPUVideo();
-  const auto& sdvType = sdv.type();
-  if (sdvType ==
-      layers::SurfaceDescriptorGPUVideo::TSurfaceDescriptorRemoteDecoder) {
-    auto& sdrd = sdv.get_SurfaceDescriptorRemoteDecoder();
-    auto& subdesc = sdrd.subdesc();
-    const auto& subdescType = subdesc.type();
-    if (subdescType == layers::RemoteDecoderVideoSubDescriptor::Tnull_t) {
-      return sd;
-    }
-    if (subdescType == layers::RemoteDecoderVideoSubDescriptor::
-                           TSurfaceDescriptorMacIOSurface) {
-      return sd;
-    }
-    if (subdescType ==
-            layers::RemoteDecoderVideoSubDescriptor::TSurfaceDescriptorD3D10 &&
-        StaticPrefs::gfx_canvas_remote_use_draw_image_fast_path_d3d()) {
-      auto& descD3D10 = subdesc.get_SurfaceDescriptorD3D10();
-      // Clear FileHandleWrapper, since FileHandleWrapper::mHandle could not be
-      // cross process delivered by using Shmem. Cross-process delivery of
-      // FileHandleWrapper::mHandle is not possible simply by using shmen. When
-      // it is tried, parent side process just causes crash during destroying
-      // FileHandleWrapper.
-      descD3D10.handle() = nullptr;
-      return sd;
-    }
-  }
-
-  return Nothing();
 }
 
 // drawImage(in HTMLImageElement image, in float dx, in float dy);

dom/canvas/CanvasRenderingContextHelper.h

@@ -19,6 +19,10 @@ namespace mozilla {
 
 class ErrorResult;
 
+namespace layers {
+class SurfaceDescriptor;
+}  // namespace layers
+
 namespace dom {
 
 class BlobCallback;
@@ -89,6 +93,9 @@ class CanvasRenderingContextHelper {
   nsCOMPtr<nsICanvasRenderingContextInternal> mCurrentContext;
 };
 
+Maybe<layers::SurfaceDescriptor> ValidSurfaceDescriptorForRemoteCanvas2d(
+    const layers::SurfaceDescriptor&);
+
 }  // namespace dom
 namespace CanvasUtils {
 bool GetCanvasContextType(const nsAString&, dom::CanvasContextType* const);

gfx/2d/RecordedEventImpl.h

@@ -17,6 +17,9 @@
 #include "ScaledFontBase.h"
 #include "SFNTData.h"
 
+#include "mozilla/dom/CanvasRenderingContextHelper.h"
+#include "mozilla/IntegerRange.h"
+#include "mozilla/layers/BuildConstants.h"
 #include "mozilla/layers/LayersSurfaces.h"
 
 namespace mozilla {
@@ -3247,6 +3250,35 @@ inline bool RecordedDrawSurfaceDescriptor::PlayEvent(
   return true;
 }
 
+template <class S>
+struct ElementStreamFormat<S, layers::SurfaceDescriptor> {
+  using T = layers::SurfaceDescriptor;
+
+  static void Write(S& s, const T& t) {
+    // More rigorous version is coming soon! -Kelsey
+    const auto valid = dom::ValidSurfaceDescriptorForRemoteCanvas2d(t);
+    MOZ_RELEASE_ASSERT(valid && *valid == t);
+    if (kIsDebug) {
+      // We better be able to memcpy and destroy this if we're going to send it
+      // over IPC!
+      constexpr int A_COUPLE_TIMES = 3;
+      for (const auto i : IntegerRange(A_COUPLE_TIMES)) {
+        (void)i;
+        auto copy = T{};
+        memcpy(&copy, &t, sizeof(T));
+      }
+    }
+    const auto& tValid = *valid;
+    s.write(reinterpret_cast<const char*>(&tValid), sizeof(T));
+  }
+  static void Read(S& s, T& t) {
+    s.read(reinterpret_cast<char*>(&t), sizeof(T));
+    const auto valid = dom::ValidSurfaceDescriptorForRemoteCanvas2d(t);
+    MOZ_RELEASE_ASSERT(valid && *valid == t);
+    t = *valid;
+  }
+};
+
 template <class S>
 void RecordedDrawSurfaceDescriptor::Record(S& aStream) const {
   WriteElement(aStream, mDesc);

gfx/2d/RecordingTypes.h

@@ -7,7 +7,7 @@
 #ifndef MOZILLA_GFX_RECORDINGTYPES_H_
 #define MOZILLA_GFX_RECORDINGTYPES_H_
 
-#include <ostream>
+#include <type_traits>
 #include <vector>
 
 #include "Logging.h"
@@ -18,12 +18,15 @@ namespace gfx {
 template <class S, class T>
 struct ElementStreamFormat {
   static void Write(S& aStream, const T& aElement) {
+    static_assert(std::is_trivially_copyable_v<T>);
     aStream.write(reinterpret_cast<const char*>(&aElement), sizeof(T));
   }
   static void Read(S& aStream, T& aElement) {
+    static_assert(std::is_trivially_copyable_v<T>);
     aStream.read(reinterpret_cast<char*>(&aElement), sizeof(T));
   }
 };
+
 template <class S>
 struct ElementStreamFormat<S, bool> {
   static void Write(S& aStream, const bool& aElement) {

gfx/gl/GLBlitHelper.cpp

@@ -1960,7 +1960,7 @@ inline auto MaybeFind(C& container, const K& key)
 
 std::shared_ptr<gl::Texture> GLBlitHelper::GetColorLutTex(
     const ColorLutKey& request) const {
-  if (const auto found = MaybeFind(mColorLutTexMap, request)) {
+  if (const auto found = gl::MaybeFind(mColorLutTexMap, request)) {
     return *found;  // Might be *Some(nullptr) -> nullptr!
   }