Bug 1932150 - ContentSubtreeIterator #1: Make the end container related logic in ContentSubtreeIterator more robust r=jjaschke,smaug,dom-core

This is more like a fix to the current logic. The end container could
be in a shadow tree of a shadow host, or in the light DOM of a shadow
host.

When ContentSubtreeIterator reaches to an ancestor of the end container,
it should decide whether dive to the shadow tree or not based on
the true info, not depends on a guess.

Differential Revision: https://phabricator.services.mozilla.com/D246951

dom/base/ContentIterator.cpp

@@ -952,15 +952,27 @@ void ContentSubtreeIterator::CacheInclusiveAncestorsOfEndContainer() {
       mRange, mAllowCrossShadowBoundary);
   nsIContent* endNode =
       endContainer->IsContent() ? endContainer->AsContent() : nullptr;
-  while (endNode) {
-    mInclusiveAncestorsOfEndContainer.AppendElement(endNode);
+
+  AncestorInfo info{endNode, false};
+  while (info.mAncestor) {
+    const nsINode* child = info.mAncestor;
+    mInclusiveAncestorsOfEndContainer.AppendElement(info);
     // Cross the boundary for contents in shadow tree.
     nsINode* parent = ShadowDOMSelectionHelpers::GetParentNodeInSameSelection(
-        *endNode, IterAllowCrossShadowBoundary());
+        *child, mAllowCrossShadowBoundary);
     if (!parent || !parent->IsContent()) {
       break;
     }
-    endNode = parent->AsContent();
+
+    const bool isDescendantInShadowTree =
+        IterAllowCrossShadowBoundary() && child->IsShadowRoot();
+
+    info.mAncestor = parent->AsContent();
+    // mIsDescendantInShadowTree indicates that whether child is in the
+    // shadow tree of parent or in the regular light DOM tree of parent.
+    // So that later, when info.mAncestor is reached, we can decide whether
+    // we should dive into the shadow tree.
+    info.mIsDescendantInShadowTree = isDescendantInShadowTree;
   }
 }
 
@@ -1164,20 +1176,20 @@ void ContentSubtreeIterator::Next() {
 
   NS_ASSERTION(nextNode, "No next sibling!?! This could mean deadlock!");
 
-  int32_t i = mInclusiveAncestorsOfEndContainer.IndexOf(nextNode);
+  int32_t i = mInclusiveAncestorsOfEndContainer.IndexOf(
+      nextNode, 0, InclusiveAncestorComparator());
+
   while (i != -1) {
     // as long as we are finding ancestors of the endpoint of the range,
     // dive down into their children
     ShadowRoot* root = ShadowDOMSelectionHelpers::GetShadowRoot(
-        nextNode, IterAllowCrossShadowBoundary());
-    if (!root) {
-      nextNode = nextNode->GetFirstChild();
+        nextNode, mAllowCrossShadowBoundary);
+    if (mInclusiveAncestorsOfEndContainer[i].mIsDescendantInShadowTree) {
+      nextNode = root->GetFirstChild();
     } else {
-      // If IterAllowCrossShadowBoundary() returns true, it means we should
-      // use shadow-including order for this iterator, that means the shadow
-      // root should always be iterated.
-      nextNode = IterAllowCrossShadowBoundary() ? root->GetFirstChild()
-                                                : nextNode->GetFirstChild();
+      MOZ_ASSERT(
+          !mInclusiveAncestorsOfEndContainer[i].mIsDescendantInShadowTree);
+      nextNode = nextNode->GetFirstChild();
     }
     NS_ASSERTION(nextNode, "Iterator error, expected a child node!");
 
@@ -1185,7 +1197,8 @@ void ContentSubtreeIterator::Next() {
     // down the child chain to the bottom without finding an interior node,
     // then the previous node should have been the last, which was
     // was tested at top of routine.
-    i = mInclusiveAncestorsOfEndContainer.IndexOf(nextNode);
+    i = mInclusiveAncestorsOfEndContainer.IndexOf(
+        nextNode, 0, InclusiveAncestorComparator());
   }
 
   mCurNode = nextNode;

dom/base/ContentIterator.h

@@ -97,6 +97,20 @@ class ContentIteratorBase {
       nsIContent* aRoot,
       dom::AllowRangeCrossShadowBoundary aAllowCrossShadowBoundary);
 
+  struct AncestorInfo {
+    nsIContent* mAncestor = nullptr;
+    // mIsDescendantInShadowTree is used to determine if we should go
+    // dive into the shadow tree or regular light DOM tree if mAncestor
+    // is a shadow host. It should always be false otherwise.
+    bool mIsDescendantInShadowTree = false;
+  };
+
+  class InclusiveAncestorComparator {
+   public:
+    bool Equals(const AncestorInfo& aA, const nsINode* aB) const {
+      return aA.mAncestor == aB;
+    }
+  };
   // Get the next/previous sibling of aNode, or its parent's, or grandparent's,
   // etc.  Returns null if aNode and all its ancestors have no next/previous
   // sibling.
@@ -330,7 +344,7 @@ class ContentSubtreeIterator final : public SafeContentIteratorBase {
   RefPtr<dom::AbstractRange> mRange;
 
   // See <https://dom.spec.whatwg.org/#concept-tree-inclusive-ancestor>.
-  AutoTArray<nsIContent*, 8> mInclusiveAncestorsOfEndContainer;
+  AutoTArray<AncestorInfo, 8> mInclusiveAncestorsOfEndContainer;
 
   // Whether this iterator allows to iterate nodes across shadow boundary.
   dom::AllowRangeCrossShadowBoundary mAllowCrossShadowBoundary =