Bug 1649857 - Fix crash in parent process when RDD and GPU processes crashed simultaneously. a=pascalc

When the RDD and GPU processes crashed simultaneously, we could get into
a position where we would try to recreate PVideoBridge between the RDD
and GPU processes, before bringing back up the RDD process. We should
now skip reinitializing the protocol if the RDD process is unavailable.

Original Revision: https://phabricator.services.mozilla.com/D175044

Differential Revision: https://phabricator.services.mozilla.com/D270327

dom/media/ipc/RDDProcessHost.cpp

@@ -256,7 +256,7 @@ void RDDProcessHost::DestroyProcess() {
 void RDDProcessHost::ResolvePromise() {
   MOZ_ASSERT(NS_IsMainThread());
 
-  if (!mLaunchPromiseSettled) {
+  if (mLaunchPromise && !mLaunchPromiseSettled) {
     mLaunchPromise->Resolve(true, __func__);
     mLaunchPromiseSettled = true;
   }
@@ -268,7 +268,7 @@ void RDDProcessHost::ResolvePromise() {
 void RDDProcessHost::RejectPromise() {
   MOZ_ASSERT(NS_IsMainThread());
 
-  if (!mLaunchPromiseSettled) {
+  if (mLaunchPromise && !mLaunchPromiseSettled) {
     mLaunchPromise->Reject(NS_ERROR_FAILURE, __func__);
     mLaunchPromiseSettled = true;
   }

dom/media/ipc/RDDProcessManager.cpp

@@ -129,7 +129,8 @@ RefPtr<GenericNonExclusivePromise> RDDProcessManager::LaunchRDDProcess() {
                                                        __func__);
   }
 
-  if (mLaunchRDDPromise && mProcess) {
+  if (mProcess) {
+    MOZ_DIAGNOSTIC_ASSERT(mLaunchRDDPromise);
     return mLaunchRDDPromise;
   }
 
@@ -154,7 +155,7 @@ RefPtr<GenericNonExclusivePromise> RDDProcessManager::LaunchRDDProcess() {
               NS_ERROR_NOT_AVAILABLE, __func__);
         }
 
-        if (IsRDDProcessDestroyed()) {
+        if (NS_WARN_IF(!IsRDDProcessLaunching())) {
           return GenericNonExclusivePromise::CreateAndReject(
               NS_ERROR_NOT_AVAILABLE, __func__);
         }
@@ -219,14 +220,14 @@ auto RDDProcessManager::EnsureRDDProcessAndCreateBridge(
       });
 }
 
-bool RDDProcessManager::IsRDDProcessLaunching() {
+bool RDDProcessManager::IsRDDProcessLaunching() const {
   MOZ_ASSERT(NS_IsMainThread());
   return !!mProcess && !mRDDChild;
 }
 
-bool RDDProcessManager::IsRDDProcessDestroyed() const {
+bool RDDProcessManager::IsRDDProcessAlive() const {
   MOZ_ASSERT(NS_IsMainThread());
-  return !mRDDChild && !mProcess;
+  return mRDDChild && mRDDChild->CanSend() && mProcess;
 }
 
 void RDDProcessManager::OnProcessUnexpectedShutdown(RDDProcessHost* aHost) {
@@ -261,14 +262,20 @@ void RDDProcessManager::NotifyRemoteActorDestroyed(
 
 void RDDProcessManager::DestroyProcess() {
   MOZ_ASSERT(NS_IsMainThread());
+
   if (!mProcess) {
     return;
   }
 
-  mProcess->Shutdown();
-  mProcessToken = 0;
+  // Move onto the stack to ensure we don't re-enter from a chained promise
+  // rejection on the process shutdown.
+  RDDProcessHost* process = mProcess;
   mProcess = nullptr;
+
+  process->Shutdown();
+  mProcessToken = 0;
   mRDDChild = nullptr;
+  mLaunchRDDPromise = nullptr;
   mQueuedPrefs.Clear();
 
   CrashReporter::RecordAnnotationCString(
@@ -280,7 +287,7 @@ bool RDDProcessManager::CreateContentBridge(
     ipc::Endpoint<PRemoteDecoderManagerChild>* aOutRemoteDecoderManager) {
   MOZ_ASSERT(NS_IsMainThread());
 
-  if (IsRDDProcessDestroyed()) {
+  if (NS_WARN_IF(!IsRDDProcessAlive())) {
     MOZ_LOG(sPDMLog, LogLevel::Debug,
             ("RDD shutdown before creating content bridge"));
     return false;
@@ -310,6 +317,12 @@ bool RDDProcessManager::CreateVideoBridge() {
   ipc::Endpoint<PVideoBridgeParent> parentPipe;
   ipc::Endpoint<PVideoBridgeChild> childPipe;
 
+  if (NS_WARN_IF(!IsRDDProcessAlive())) {
+    MOZ_LOG(sPDMLog, LogLevel::Debug,
+            ("RDD shutdown before creating video bridge"));
+    return false;
+  }
+
   GPUProcessManager* gpuManager = GPUProcessManager::Get();
   ipc::EndpointProcInfo gpuProcessInfo = gpuManager
                                              ? gpuManager->GPUEndpointProcInfo()

dom/media/ipc/RDDProcessManager.h

@@ -74,8 +74,8 @@ class RDDProcessManager final : public RDDProcessHost::Listener {
   RefPtr<PRDDChild::TestTriggerMetricsPromise> TestTriggerMetrics();
 
  private:
-  bool IsRDDProcessLaunching();
-  bool IsRDDProcessDestroyed() const;
+  bool IsRDDProcessLaunching() const;
+  bool IsRDDProcessAlive() const;
   bool CreateVideoBridge();
 
   // Called from our xpcom-shutdown observer.