Bug 1988912. r=ahale a=dmeehan

Differential Revision: https://phabricator.services.mozilla.com/D265351

dom/canvas/CanvasRenderingContext2D.cpp

@@ -5503,20 +5503,20 @@ static Matrix ComputeRotationMatrix(gfxFloat aRotatedWidth,
 
 // -
 
-Maybe<layers::SurfaceDescriptor> ValidSurfaceDescriptorForRemoteCanvas2d(
-    const layers::SurfaceDescriptor& sdConst) {
-  auto sd = sdConst;  // Copy, so we can mutate it.
-  if (sd.type() != layers::SurfaceDescriptor::TSurfaceDescriptorGPUVideo) {
-    return Nothing();
+bool ValidSurfaceDescriptorForRemoteCanvas2d(
+    const layers::SurfaceDescriptor& aSd,
+    Maybe<layers::SurfaceDescriptor>* aResultSd) {
+  if (aSd.type() != layers::SurfaceDescriptor::TSurfaceDescriptorGPUVideo) {
+    return false;
   }
 
-  auto& sdv = sd.get_SurfaceDescriptorGPUVideo();
+  const auto& sdv = aSd.get_SurfaceDescriptorGPUVideo();
   if (sdv.type() !=
       layers::SurfaceDescriptorGPUVideo::TSurfaceDescriptorRemoteDecoder) {
-    return Nothing();
+    return false;
   }
-  auto& sdrd = sdv.get_SurfaceDescriptorRemoteDecoder();
-  auto& subdesc = sdrd.subdesc();
+  const auto& sdrd = sdv.get_SurfaceDescriptorRemoteDecoder();
+  const auto& subdesc = sdrd.subdesc();
   switch (subdesc.type()) {
     case layers::RemoteDecoderVideoSubDescriptor::Tnull_t:
       break;
@@ -5525,7 +5525,7 @@ Maybe<layers::SurfaceDescriptor> ValidSurfaceDescriptorForRemoteCanvas2d(
         TSurfaceDescriptorMacIOSurface: {
       const auto& ssd = subdesc.get_SurfaceDescriptorMacIOSurface();
       if (ssd.gpuFence()) {
-        return Nothing();
+        return false;
       }
       break;
     }
@@ -5533,18 +5533,31 @@ Maybe<layers::SurfaceDescriptor> ValidSurfaceDescriptorForRemoteCanvas2d(
 #ifdef XP_WIN
     case layers::RemoteDecoderVideoSubDescriptor::TSurfaceDescriptorD3D10: {
       if (!StaticPrefs::gfx_canvas_remote_use_draw_image_fast_path_d3d()) {
-        return Nothing();
+        return false;
       }
-      auto& ssd = subdesc.get_SurfaceDescriptorD3D10();
-      ssd.handle() =
-          nullptr;  // Not IPC-able, but it's just an optimization to have this.
-      break;
+      const auto& ssd = subdesc.get_SurfaceDescriptorD3D10();
+      if (aResultSd) {
+        *aResultSd = Some(aSd);
+        // Not IPC-able, but it's just an optimization to have this.
+        aResultSd->ref()
+            .get_SurfaceDescriptorGPUVideo()
+            .get_SurfaceDescriptorRemoteDecoder()
+            .subdesc()
+            .get_SurfaceDescriptorD3D10()
+            .handle() = nullptr;
+      } else if (ssd.handle()) {
+        return false;
+      }
+      return true;
     }
 #endif
     default:
-      return Nothing();
+      return false;
   }
-  return Some(sd);
+  if (aResultSd) {
+    *aResultSd = Some(aSd);
+  }
+  return true;
 }
 
 static Maybe<layers::SurfaceDescriptor>
@@ -5558,9 +5571,13 @@ MaybeGetSurfaceDescriptorForRemoteCanvas(
     return Nothing();
   }
 
-  const auto sd = aResult.mLayersImage->GetDesc();
-  if (!sd) return Nothing();
-  return ValidSurfaceDescriptorForRemoteCanvas2d(*sd);
+  if (const auto sd = aResult.mLayersImage->GetDesc()) {
+    Maybe<layers::SurfaceDescriptor> result;
+    if (ValidSurfaceDescriptorForRemoteCanvas2d(*sd, &result)) {
+      return result;
+    }
+  }
+  return Nothing();
 }
 
 // drawImage(in HTMLImageElement image, in float dx, in float dy);

dom/canvas/CanvasRenderingContextHelper.h

@@ -89,8 +89,9 @@ class CanvasRenderingContextHelper {
   nsCOMPtr<nsICanvasRenderingContextInternal> mCurrentContext;
 };
 
-Maybe<layers::SurfaceDescriptor> ValidSurfaceDescriptorForRemoteCanvas2d(
-    const layers::SurfaceDescriptor&);
+bool ValidSurfaceDescriptorForRemoteCanvas2d(
+    const layers::SurfaceDescriptor& aSd,
+    Maybe<layers::SurfaceDescriptor>* aResultSd = nullptr);
 
 }  // namespace dom
 namespace CanvasUtils {

gfx/2d/RecordedEventImpl.h

@@ -3255,8 +3255,10 @@ struct ElementStreamFormat<S, layers::SurfaceDescriptor> {
   using T = layers::SurfaceDescriptor;
 
   static void Write(S& s, const T& t) {
-    // More rigorous version is coming soon! -Kelsey
-    const auto valid = dom::ValidSurfaceDescriptorForRemoteCanvas2d(t);
+    Maybe<T> valid;
+    if (!dom::ValidSurfaceDescriptorForRemoteCanvas2d(t, &valid)) {
+      MOZ_CRASH("Invalid surface descriptor for write");
+    }
     MOZ_RELEASE_ASSERT(valid && *valid == t);
     if (kIsDebug) {
       // We better be able to memcpy and destroy this if we're going to send it
@@ -3272,10 +3274,15 @@ struct ElementStreamFormat<S, layers::SurfaceDescriptor> {
     s.write(reinterpret_cast<const char*>(&tValid), sizeof(T));
   }
   static void Read(S& s, T& t) {
-    s.read(reinterpret_cast<char*>(&t), sizeof(T));
-    const auto valid = dom::ValidSurfaceDescriptorForRemoteCanvas2d(t);
-    MOZ_RELEASE_ASSERT(valid && *valid == t);
-    t = *valid;
+    char buf[sizeof(T)];
+    s.read(buf, sizeof(T));
+    const auto& sd = *reinterpret_cast<const layers::SurfaceDescriptor*>(buf);
+    if (dom::ValidSurfaceDescriptorForRemoteCanvas2d(sd)) {
+      t = sd;
+      MOZ_RELEASE_ASSERT(sd == t);
+    } else {
+      s.SetIsBad();
+    }
   }
 };