Bug 594774. Detect DOM changes during painting, and abort painting ASAP when that happens. r=dbaron a=blocking

layout/base/FrameLayerBuilder.cpp

@@ -44,6 +44,7 @@
 #include "BasicLayers.h"
 #include "nsSubDocumentFrame.h"
 #include "nsCSSRendering.h"
+#include "nsCSSFrameConstructor.h"
 
 #ifdef DEBUG
 #include <stdio.h>
@@ -360,6 +361,15 @@ PRUint8 gLayerManagerUserData;
 
 } // anonymous namespace
 
+void
+FrameLayerBuilder::Init(nsDisplayListBuilder* aBuilder)
+{
+  mRootPresContext = aBuilder->ReferenceFrame()->PresContext()->GetRootPresContext();
+  if (mRootPresContext) {
+    mInitialDOMGeneration = mRootPresContext->GetDOMGeneration();
+  }
+}
+
 PRBool
 FrameLayerBuilder::DisplayItemDataEntry::HasContainerLayer()
 {
@@ -1540,6 +1550,10 @@ FrameLayerBuilder::DrawThebesLayer(ThebesLayer* aLayer,
 {
   nsDisplayListBuilder* builder = static_cast<nsDisplayListBuilder*>
     (aCallbackData);
+
+  if (builder->LayerBuilder()->CheckDOMModified())
+    return;
+
   nsTArray<ClippedDisplayItem> items;
   nsIFrame* containerLayerFrame;
   {
@@ -1669,6 +1683,9 @@ FrameLayerBuilder::DrawThebesLayer(ThebesLayer* aLayer,
     } else {
       cdi->mItem->Paint(builder, rc);
     }
+
+    if (builder->LayerBuilder()->CheckDOMModified())
+      break;
   }
 
   if (setClipRect) {
@@ -1676,6 +1693,24 @@ FrameLayerBuilder::DrawThebesLayer(ThebesLayer* aLayer,
   }
 }
 
+PRBool
+FrameLayerBuilder::CheckDOMModified()
+{
+  if (mRootPresContext &&
+      mInitialDOMGeneration == mRootPresContext->GetDOMGeneration())
+    return PR_FALSE;
+  if (mDetectedDOMModification) {
+    // Don't spam the console with extra warnings
+    return PR_TRUE;
+  }
+  mDetectedDOMModification = PR_TRUE;
+  // Painting is not going to complete properly. There's not much
+  // we can do here though. Invalidating the window to get another repaint
+  // is likely to lead to an infinite repaint loop.
+  NS_WARNING("Detected DOM modification during paint, bailing out!");
+  return PR_TRUE;
+}
+
 #ifdef DEBUG
 void
 FrameLayerBuilder::DumpRetainedLayerTree()

layout/base/FrameLayerBuilder.h

@@ -49,6 +49,7 @@ class nsDisplayListBuilder;
 class nsDisplayList;
 class nsDisplayItem;
 class gfxContext;
+class nsRootPresContext;
 
 namespace mozilla {
 
@@ -97,6 +98,7 @@ public:
 
   FrameLayerBuilder() :
     mRetainingManager(nsnull),
+    mDetectedDOMModification(PR_FALSE),
     mInvalidateAllThebesContent(PR_FALSE),
     mInvalidateAllLayers(PR_FALSE)
   {
@@ -104,6 +106,8 @@ public:
     mThebesLayerItems.Init();
   }
 
+  void Init(nsDisplayListBuilder* aBuilder);
+
   /**
    * Call this to notify that we are about to start a transaction on the
    * retained layer manager aManager.
@@ -411,11 +415,22 @@ protected:
   static PLDHashOperator StoreNewDisplayItemData(DisplayItemDataEntry* aEntry,
                                                  void* aUserArg);
 
+  /**
+   * Returns true if the DOM has been modified since we started painting,
+   * in which case we should bail out and not paint anymore. This should
+   * never happen, but plugins can trigger it in some cases.
+   */
+  PRBool CheckDOMModified();
+
   /**
    * The layer manager belonging to the widget that is being retained
    * across paints.
    */
   LayerManager*                       mRetainingManager;
+  /**
+   * The root prescontext for the display list builder reference frame
+   */
+  nsRootPresContext*                  mRootPresContext;
   /**
    * A map from frames to a list of (display item key, layer) pairs that
    * describes what layers various parts of the frame are assigned to.
@@ -426,6 +441,15 @@ protected:
    * clipping data) to be rendered in the layer.
    */
   nsTHashtable<ThebesLayerItemsEntry> mThebesLayerItems;
+  /**
+   * Saved generation counter so we can detect DOM changes.
+   */
+  PRUint32                            mInitialDOMGeneration;
+  /**
+   * Set to true if we have detected and reported DOM modification during
+   * the current paint.
+   */
+  PRPackedBool                        mDetectedDOMModification;
   /**
    * Indicates that the contents of all ThebesLayers should be rerendered
    * during this paint.

layout/base/nsCSSFrameConstructor.cpp

@@ -8188,6 +8188,12 @@ nsCSSFrameConstructor::BeginUpdate() {
   NS_ASSERTION(!nsContentUtils::IsSafeToRunScript(),
                "Someone forgot a script blocker");
 
+  nsRootPresContext* rootPresContext =
+    mPresShell->GetPresContext()->GetRootPresContext();
+  if (rootPresContext) {
+    rootPresContext->IncrementDOMGeneration();
+  }
+
   ++mUpdateCount;
 }
 

layout/base/nsDisplayList.cpp

@@ -96,6 +96,8 @@ nsDisplayListBuilder::nsDisplayListBuilder(nsIFrame* aReferenceFrame,
     }
   }
 
+  LayerBuilder()->Init(this);
+
   PR_STATIC_ASSERT(nsDisplayItem::TYPE_MAX < (1 << nsDisplayItem::TYPE_BITS));
 }
 

layout/base/nsPresContext.cpp

@@ -2423,6 +2423,7 @@ nsRootPresContext::nsRootPresContext(nsIDocument* aDocument,
                                      nsPresContextType aType)
   : nsPresContext(aDocument, aType),
     mUpdatePluginGeometryForFrame(nsnull),
+    mDOMGeneration(0),
     mNeedsToUpdatePluginGeometry(PR_FALSE)
 {
   mRegisteredPlugins.Init();

layout/base/nsPresContext.h

@@ -1267,12 +1267,25 @@ public:
    */
   void RootForgetUpdatePluginGeometryFrame(nsIFrame* aFrame);
 
+  /**
+   * Increment DOM-modification generation counter to indicate that
+   * the DOM has changed in a way that might lead to style changes/
+   * reflows/frame creation and destruction.
+   */
+  void IncrementDOMGeneration() { mDOMGeneration++; }
+
+  /**
+   * Get the current DOM generation counter.
+   */
+  PRUint32 GetDOMGeneration() { return mDOMGeneration; }
+
 private:
   nsTHashtable<nsPtrHashKey<nsObjectFrame> > mRegisteredPlugins;
   // if mNeedsToUpdatePluginGeometry is set, then this is the frame to
   // use as the root of the subtree to search for plugin updates, or
   // null to use the root frame of this prescontext
   nsIFrame* mUpdatePluginGeometryForFrame;
+  PRUint32 mDOMGeneration;
   PRPackedBool mNeedsToUpdatePluginGeometry;
 };