Bug 1898142 - Deliver image usage type to WebRenderImageHost r=gfx-reviewers,lsalzman

Adds ImageUsageType to ImageClient and ImageContainer to identify user of Image at WebRenderImageHost.

Some ImageContainers are used only for allocating Image. Only following types calls ImageContainer::SetCurrentImages().
- ImageUsageType::Canvas
- ImageUsageType::OffscreenCanvas
- ImageUsageType::VideoFrameContainer

Differential Revision: https://phabricator.services.mozilla.com/D211147

dom/canvas/OffscreenCanvasDisplayHelper.cpp

@@ -92,7 +92,9 @@ void OffscreenCanvasDisplayHelper::UpdateContext(
     OffscreenCanvas* aOffscreenCanvas, RefPtr<ThreadSafeWorkerRef>&& aWorkerRef,
     CanvasContextType aType, const Maybe<int32_t>& aChildId) {
   RefPtr<layers::ImageContainer> imageContainer =
-      MakeRefPtr<layers::ImageContainer>(layers::ImageContainer::ASYNCHRONOUS);
+      MakeRefPtr<layers::ImageContainer>(
+          layers::ImageUsageType::OffscreenCanvas,
+          layers::ImageContainer::ASYNCHRONOUS);
 
   MutexAutoLock lock(mMutex);
 

dom/html/HTMLMediaElement.cpp

@@ -6278,7 +6278,8 @@ VideoFrameContainer* HTMLMediaElement::GetVideoFrameContainer() {
   }
 
   mVideoFrameContainer = new VideoFrameContainer(
-      this, MakeAndAddRef<ImageContainer>(ImageContainer::ASYNCHRONOUS));
+      this, MakeAndAddRef<ImageContainer>(ImageUsageType::VideoFrameContainer,
+                                          ImageContainer::ASYNCHRONOUS));
 
   return mVideoFrameContainer;
 }

dom/media/VideoSegment.cpp

@@ -44,8 +44,8 @@ void VideoFrame::TakeFrom(VideoFrame* aFrame) {
 /* static */
 already_AddRefed<Image> VideoFrame::CreateBlackImage(
     const gfx::IntSize& aSize) {
-  RefPtr<ImageContainer> container =
+  RefPtr<ImageContainer> container = MakeAndAddRef<ImageContainer>(
-      MakeAndAddRef<ImageContainer>(ImageContainer::ASYNCHRONOUS);
+      ImageUsageType::BlackImage, ImageContainer::ASYNCHRONOUS);
   RefPtr<PlanarYCbCrImage> image = container->CreatePlanarYCbCrImage();
   if (!image) {
     return nullptr;

dom/media/ipc/RemoteVideoDecoder.cpp

@@ -152,7 +152,9 @@ RemoteVideoDecoderParent::RemoteVideoDecoderParent(
 
 IPCResult RemoteVideoDecoderParent::RecvConstruct(
     ConstructResolver&& aResolver) {
-  auto imageContainer = MakeRefPtr<layers::ImageContainer>();
+  auto imageContainer = MakeRefPtr<layers::ImageContainer>(
+      layers::ImageUsageType::RemoteVideoDecoder,
+      layers::ImageContainer::SYNCHRONOUS);
   if (mKnowsCompositor && XRE_IsRDDProcess()) {
     // Ensure to allocate recycle allocator
     imageContainer->EnsureRecycleAllocatorForRDD(mKnowsCompositor);

dom/media/webcodecs/DecoderAgent.cpp

@@ -51,6 +51,7 @@ DecoderAgent::DecoderAgent(Id aId, UniquePtr<TrackInfo>&& aInfo)
       mOwnerThread(GetCurrentSerialEventTarget()),
       mPDMFactory(MakeRefPtr<PDMFactory>()),
       mImageContainer(MakeAndAddRef<layers::ImageContainer>(
+          layers::ImageUsageType::WebCodecs,
           layers::ImageContainer::ASYNCHRONOUS)),
       mDecoder(nullptr),
       mState(State::Unconfigured) {

dom/media/webrtc/MediaEngineFake.cpp

@@ -299,7 +299,7 @@ nsresult MediaEngineFakeVideoSource::Start() {
 
   if (!mImageContainer) {
     mImageContainer = MakeAndAddRef<layers::ImageContainer>(
-        layers::ImageContainer::ASYNCHRONOUS);
+        layers::ImageUsageType::Webrtc, layers::ImageContainer::ASYNCHRONOUS);
   }
 
   // Start timer for subsequent frames

dom/media/webrtc/MediaEngineRemoteVideoSource.cpp

@@ -202,7 +202,7 @@ void MediaEngineRemoteVideoSource::SetTrack(const RefPtr<MediaTrack>& aTrack,
 
   if (!mImageContainer) {
     mImageContainer = MakeAndAddRef<layers::ImageContainer>(
-        layers::ImageContainer::ASYNCHRONOUS);
+        layers::ImageUsageType::Webrtc, layers::ImageContainer::ASYNCHRONOUS);
   }
 
   {

dom/media/webrtc/libwebrtcglue/WebrtcMediaDataDecoderCodec.cpp

@@ -20,6 +20,7 @@ WebrtcMediaDataDecoder::WebrtcMediaDataDecoder(nsACString& aCodecMimeType,
       mTaskQueue(TaskQueue::Create(do_AddRef(mThreadPool),
                                    "WebrtcMediaDataDecoder::mTaskQueue")),
       mImageContainer(MakeAndAddRef<layers::ImageContainer>(
+          layers::ImageUsageType::Webrtc,
           layers::ImageContainer::ASYNCHRONOUS)),
       mFactory(new PDMFactory()),
       mTrackType(TrackInfo::kUndefinedTrack),

dom/media/webrtc/transportbridge/MediaPipeline.cpp

@@ -1398,8 +1398,8 @@ class MediaPipelineReceiveVideo::PipelineListener
   PipelineListener(RefPtr<SourceMediaTrack> aSource, TrackingId aTrackingId,
                    PrincipalHandle aPrincipalHandle, PrincipalPrivacy aPrivacy)
       : GenericReceiveListener(std::move(aSource), std::move(aTrackingId)),
-        mImageContainer(
+        mImageContainer(MakeAndAddRef<ImageContainer>(
-            MakeAndAddRef<ImageContainer>(ImageContainer::ASYNCHRONOUS)),
+            ImageUsageType::Webrtc, ImageContainer::ASYNCHRONOUS)),
         mMutex("MediaPipelineReceiveVideo::PipelineListener::mMutex"),
         mPrincipalHandle(std::move(aPrincipalHandle)),
         mPrivacy(aPrivacy) {}

gfx/layers/CompositorTypes.h

@@ -148,6 +148,20 @@ enum class CompositableType : uint8_t {
   COUNT
 };
 
+enum class ImageUsageType : uint8_t {
+  UNKNOWN,
+  WebRenderImageData,
+  WebRenderFallbackData,
+  Canvas,
+  OffscreenCanvas,
+  VideoFrameContainer,
+  RemoteVideoDecoder,
+  BlackImage,
+  Webrtc,
+  WebCodecs,
+  COUNT
+};
+
 /**
  * Sent from the compositor to the content-side LayerManager, includes
  * properties of the compositor and should (in the future) include information
@@ -223,15 +237,19 @@ struct TextureFactoryIdentifier {
  */
 struct TextureInfo {
   CompositableType mCompositableType;
+  ImageUsageType mUsageType;
   TextureFlags mTextureFlags;
 
   TextureInfo()
       : mCompositableType(CompositableType::UNKNOWN),
+        mUsageType(ImageUsageType::UNKNOWN),
         mTextureFlags(TextureFlags::NO_FLAGS) {}
 
-  explicit TextureInfo(CompositableType aType,
+  TextureInfo(CompositableType aType, ImageUsageType aUsageType,
-                       TextureFlags aTextureFlags = TextureFlags::DEFAULT)
+              TextureFlags aTextureFlags)
-      : mCompositableType(aType), mTextureFlags(aTextureFlags) {}
+      : mCompositableType(aType),
+        mUsageType(aUsageType),
+        mTextureFlags(aTextureFlags) {}
 
   bool operator==(const TextureInfo& aOther) const {
     return mCompositableType == aOther.mCompositableType &&

gfx/layers/ImageContainer.cpp

@@ -163,30 +163,32 @@ void ImageContainer::EnsureImageClient() {
   }
 
   RefPtr<ImageBridgeChild> imageBridge = ImageBridgeChild::GetSingleton();
-  if (imageBridge) {
+  if (!imageBridge) {
-    mImageClient =
+    return;
-        imageBridge->CreateImageClient(CompositableType::IMAGE, this);
+  }
-    if (mImageClient) {
+
-      mAsyncContainerHandle = mImageClient->GetAsyncHandle();
+  mImageClient = imageBridge->CreateImageClient(CompositableType::IMAGE, this);
-    } else {
+  if (mImageClient) {
-      // It's okay to drop the async container handle since the ImageBridgeChild
+    mAsyncContainerHandle = mImageClient->GetAsyncHandle();
-      // is going to die anyway.
+  } else {
-      mAsyncContainerHandle = CompositableHandle();
+    // It's okay to drop the async container handle since the ImageBridgeChild
-    }
+    // is going to die anyway.
+    mAsyncContainerHandle = CompositableHandle();
   }
 }
 
-ImageContainer::ImageContainer(Mode flag)
+ImageContainer::ImageContainer(ImageUsageType aUsageType, Mode aFlag)
-    : mRecursiveMutex("ImageContainer.mRecursiveMutex"),
+    : mUsageType(aUsageType),
+      mIsAsync(aFlag == ASYNCHRONOUS),
+      mRecursiveMutex("ImageContainer.mRecursiveMutex"),
       mGenerationCounter(++sGenerationCounter),
       mPaintCount(0),
       mDroppedImageCount(0),
       mImageFactory(new ImageFactory()),
       mRotation(VideoRotation::kDegree_0),
       mRecycleBin(new BufferRecycleBin()),
-      mIsAsync(flag == ASYNCHRONOUS),
       mCurrentProducerID(-1) {
-  if (flag == ASYNCHRONOUS) {
+  if (aFlag == ASYNCHRONOUS) {
     mNotifyCompositeListener = new ImageContainerListener(this);
     EnsureImageClient();
   }
@@ -350,6 +352,10 @@ void ImageContainer::ClearImagesFromImageBridge() {
 void ImageContainer::SetCurrentImages(const nsTArray<NonOwningImage>& aImages) {
   AUTO_PROFILER_LABEL("ImageContainer::SetCurrentImages", GRAPHICS);
   MOZ_ASSERT(!aImages.IsEmpty());
+  MOZ_ASSERT(mUsageType == ImageUsageType::Canvas ||
+             mUsageType == ImageUsageType::OffscreenCanvas ||
+             mUsageType == ImageUsageType::VideoFrameContainer);
+
   RecursiveMutexAutoLock lock(mRecursiveMutex);
   if (mIsAsync) {
     if (RefPtr<ImageBridgeChild> imageBridge =
@@ -399,6 +405,9 @@ void ImageContainer::SetCurrentImageInTransaction(Image* aImage) {
 
 void ImageContainer::SetCurrentImagesInTransaction(
     const nsTArray<NonOwningImage>& aImages) {
+  MOZ_ASSERT(!mIsAsync);
+  MOZ_ASSERT(mUsageType == ImageUsageType::WebRenderFallbackData);
+
   NS_ASSERTION(NS_IsMainThread(), "Should be on main thread.");
   NS_ASSERTION(!HasImageClient(),
                "Should use async image transfer with ImageBridge.");

gfx/layers/ImageContainer.h

@@ -324,7 +324,7 @@ class ImageContainer final : public SupportsThreadSafeWeakPtr<ImageContainer> {
 
   static const uint64_t sInvalidAsyncContainerId = 0;
 
-  explicit ImageContainer(ImageContainer::Mode flag = SYNCHRONOUS);
+  ImageContainer(ImageUsageType aUsageType, ImageContainer::Mode aFlag);
 
   ~ImageContainer();
 
@@ -579,6 +579,9 @@ class ImageContainer final : public SupportsThreadSafeWeakPtr<ImageContainer> {
 
   void DropImageClient();
 
+  const ImageUsageType mUsageType;
+  const bool mIsAsync;
+
  private:
   typedef mozilla::RecursiveMutex RecursiveMutex;
 
@@ -655,7 +658,6 @@ class ImageContainer final : public SupportsThreadSafeWeakPtr<ImageContainer> {
   // than asynchronusly using the ImageBridge IPDL protocol.
   RefPtr<ImageClient> mImageClient MOZ_GUARDED_BY(mRecursiveMutex);
 
-  const bool mIsAsync;
   CompositableHandle mAsyncContainerHandle MOZ_GUARDED_BY(mRecursiveMutex);
 
   // ProducerID for last current image(s)

gfx/layers/client/CanvasClient.h

@@ -54,7 +54,8 @@ class CanvasClient final : public CompositableClient {
   }
 
   TextureInfo GetTextureInfo() const override {
-    return TextureInfo(CompositableType::IMAGE, mTextureFlags);
+    return TextureInfo(CompositableType::IMAGE, ImageUsageType::Canvas,
+                       mTextureFlags);
   }
 
   void OnDetach() override { Clear(); }

gfx/layers/client/ImageClient.cpp

@@ -40,13 +40,13 @@ using namespace mozilla::gfx;
 
 /* static */
 already_AddRefed<ImageClient> ImageClient::CreateImageClient(
-    CompositableType aCompositableHostType, CompositableForwarder* aForwarder,
+    CompositableType aCompositableHostType, ImageUsageType aUsageType,
-    TextureFlags aFlags) {
+    CompositableForwarder* aForwarder, TextureFlags aFlags) {
   RefPtr<ImageClient> result = nullptr;
   switch (aCompositableHostType) {
     case CompositableType::IMAGE:
-      result =
+      result = new ImageClientSingle(aForwarder, aFlags,
-          new ImageClientSingle(aForwarder, aFlags, CompositableType::IMAGE);
+                                     CompositableType::IMAGE, aUsageType);
       break;
     case CompositableType::UNKNOWN:
       result = nullptr;
@@ -66,11 +66,13 @@ void ImageClient::RemoveTexture(TextureClient* aTexture) {
 
 ImageClientSingle::ImageClientSingle(CompositableForwarder* aFwd,
                                      TextureFlags aFlags,
-                                     CompositableType aType)
+                                     CompositableType aType,
-    : ImageClient(aFwd, aFlags, aType) {}
+                                     ImageUsageType aUsageType)
+    : ImageClient(aFwd, aFlags, aType, aUsageType) {}
 
 TextureInfo ImageClientSingle::GetTextureInfo() const {
-  return TextureInfo(CompositableType::IMAGE);
+  return TextureInfo(CompositableType::IMAGE, mUsageType,
+                     TextureFlags::DEFAULT);
 }
 
 void ImageClientSingle::FlushAllImages() {
@@ -270,9 +272,10 @@ bool ImageClientSingle::AddTextureClient(TextureClient* aTexture) {
 void ImageClientSingle::OnDetach() { mBuffers.Clear(); }
 
 ImageClient::ImageClient(CompositableForwarder* aFwd, TextureFlags aFlags,
-                         CompositableType aType)
+                         CompositableType aType, ImageUsageType aUsageType)
     : CompositableClient(aFwd, aFlags),
       mType(aType),
+      mUsageType(aUsageType),
       mLastUpdateGenerationCounter(0) {}
 
 }  // namespace layers

gfx/layers/client/ImageClient.h

@@ -41,8 +41,8 @@ class ImageClient : public CompositableClient {
    * host.
    */
   static already_AddRefed<ImageClient> CreateImageClient(
-      CompositableType aImageHostType, CompositableForwarder* aFwd,
+      CompositableType aImageHostType, ImageUsageType aUsageType,
-      TextureFlags aFlags);
+      CompositableForwarder* aFwd, TextureFlags aFlags);
 
   virtual ~ImageClient() = default;
 
@@ -72,11 +72,13 @@ class ImageClient : public CompositableClient {
 
   virtual RefPtr<TextureClient> GetForwardedTexture() { return nullptr; }
 
+  CompositableType mType;
+  ImageUsageType mUsageType;
+
  protected:
   ImageClient(CompositableForwarder* aFwd, TextureFlags aFlags,
-              CompositableType aType);
+              CompositableType aType, ImageUsageType aUsageType);
 
-  CompositableType mType;
   uint32_t mLastUpdateGenerationCounter;
 };
 
@@ -86,7 +88,7 @@ class ImageClient : public CompositableClient {
 class ImageClientSingle : public ImageClient {
  public:
   ImageClientSingle(CompositableForwarder* aFwd, TextureFlags aFlags,
-                    CompositableType aType);
+                    CompositableType aType, ImageUsageType aUsageType);
 
   bool UpdateImage(ImageContainer* aContainer) override;
 

gfx/layers/composite/CompositableHost.h

@@ -100,9 +100,9 @@ class CompositableHost {
 
   virtual uint32_t GetDroppedFrames() { return 0; }
 
+  const TextureInfo mTextureInfo;
+
  protected:
- protected:
-  TextureInfo mTextureInfo;
   AsyncCompositableRef mAsyncRef;
 };
 

gfx/layers/ipc/ImageBridgeChild.cpp

@@ -681,8 +681,8 @@ RefPtr<ImageClient> ImageBridgeChild::CreateImageClientNow(
     return nullptr;
   }
 
-  RefPtr<ImageClient> client =
+  RefPtr<ImageClient> client = ImageClient::CreateImageClient(
-      ImageClient::CreateImageClient(aType, this, TextureFlags::NO_FLAGS);
+      aType, aImageContainer->mUsageType, this, TextureFlags::NO_FLAGS);
   MOZ_ASSERT(client, "failed to create ImageClient");
   if (client) {
     client->Connect(aImageContainer);

gfx/layers/ipc/LayersMessageUtils.h

@@ -668,11 +668,13 @@ struct ParamTraits<mozilla::layers::TextureInfo> {
 
   static void Write(MessageWriter* aWriter, const paramType& aParam) {
     WriteParam(aWriter, aParam.mCompositableType);
+    WriteParam(aWriter, aParam.mUsageType);
     WriteParam(aWriter, aParam.mTextureFlags);
   }
 
   static bool Read(MessageReader* aReader, paramType* aResult) {
     return ReadParam(aReader, &aResult->mCompositableType) &&
+           ReadParam(aReader, &aResult->mUsageType) &&
            ReadParam(aReader, &aResult->mTextureFlags);
   }
 };
@@ -684,6 +686,13 @@ struct ParamTraits<mozilla::layers::CompositableType>
           mozilla::layers::CompositableType::UNKNOWN,
           mozilla::layers::CompositableType::COUNT> {};
 
+template <>
+struct ParamTraits<mozilla::layers::ImageUsageType>
+    : public ContiguousEnumSerializer<mozilla::layers::ImageUsageType,
+                                      mozilla::layers::ImageUsageType::UNKNOWN,
+                                      mozilla::layers::ImageUsageType::COUNT> {
+};
+
 template <>
 struct ParamTraits<mozilla::layers::ScrollableLayerGuid> {
   typedef mozilla::layers::ScrollableLayerGuid paramType;

gfx/layers/wr/WebRenderCommandBuilder.cpp

@@ -2633,7 +2633,8 @@ WebRenderCommandBuilder::GenerateFallbackData(
 
       imageData->CreateImageClientIfNeeded();
       RefPtr<ImageClient> imageClient = imageData->GetImageClient();
-      RefPtr<ImageContainer> imageContainer = MakeAndAddRef<ImageContainer>();
+      RefPtr<ImageContainer> imageContainer = MakeAndAddRef<ImageContainer>(
+          ImageUsageType::WebRenderFallbackData, ImageContainer::SYNCHRONOUS);
 
       {
         UpdateImageHelper helper(imageContainer, imageClient,

gfx/layers/wr/WebRenderUserData.cpp

@@ -235,7 +235,8 @@ void WebRenderImageData::CreateAsyncImageWebRenderCommands(
 void WebRenderImageData::CreateImageClientIfNeeded() {
   if (!mImageClient) {
     mImageClient = ImageClient::CreateImageClient(
-        CompositableType::IMAGE, WrBridge(), TextureFlags::DEFAULT);
+        CompositableType::IMAGE, ImageUsageType::WebRenderImageData, WrBridge(),
+        TextureFlags::DEFAULT);
     if (!mImageClient) {
       return;
     }
@@ -399,7 +400,8 @@ void WebRenderCanvasData::SetImageContainer(ImageContainer* aImageContainer) {
 
 ImageContainer* WebRenderCanvasData::GetImageContainer() {
   if (!mContainer) {
-    mContainer = MakeAndAddRef<ImageContainer>();
+    mContainer = MakeAndAddRef<ImageContainer>(ImageUsageType::Canvas,
+                                               ImageContainer::SYNCHRONOUS);
   }
   return mContainer;
 }