Bug 756165 - Return close code 1015 to JS when TLS setup fails, r=baku, r=valentin

We cannot simply pass the status code using nsIWebSocketListener::OnServerClose because it's called only when the connection is established. Instead, I'm passing TLS failure from http channel to nsIWebSocketListener::OnStop where the correct status code is set.

dom/websocket/WebSocket.cpp

@@ -820,6 +820,11 @@ WebSocketImpl::ScheduleConnectionCloseEvents(nsISupports* aContext,
       aStatusCode = NS_OK;
     }
 
+    if (aStatusCode == NS_ERROR_NET_INADEQUATE_SECURITY) {
+      // TLS negotiation failed so we need to set status code to 1015.
+      mCloseEventCode = 1015;
+    }
+
     if (NS_FAILED(aStatusCode)) {
       ConsoleError();
       mFailed = true;

netwerk/protocol/websocket/WebSocketChannel.cpp

@@ -59,6 +59,7 @@
 #include "mozilla/Telemetry.h"
 #include "mozilla/TimeStamp.h"
 #include "nsSocketTransportService2.h"
+#include "nsINSSErrorsService.h"
 
 #include "plbase64.h"
 #include "prmem.h"
@@ -3817,9 +3818,25 @@ WebSocketChannel::OnStartRequest(nsIRequest *aRequest,
 
   rv = mHttpChannel->GetResponseStatus(&status);
   if (NS_FAILED(rv)) {
+    nsresult httpStatus;
+    rv = NS_ERROR_CONNECTION_REFUSED;
+
+    // If we failed to connect due to unsuccessful TLS handshake, we must
+    // propagate a specific error to mozilla::dom::WebSocketImpl so it can set
+    // status code to 1015. Otherwise return NS_ERROR_CONNECTION_REFUSED.
+    if (NS_SUCCEEDED(mHttpChannel->GetStatus(&httpStatus))) {
+      uint32_t errorClass;
+      nsCOMPtr<nsINSSErrorsService> errSvc =
+        do_GetService("@mozilla.org/nss_errors_service;1");
+      // If GetErrorClass succeeds httpStatus is TLS related failure.
+      if (errSvc && NS_SUCCEEDED(errSvc->GetErrorClass(httpStatus, &errorClass))) {
+        rv = NS_ERROR_NET_INADEQUATE_SECURITY;
+      }
+    }
+
     LOG(("WebSocketChannel::OnStartRequest: No HTTP Response\n"));
-    AbortSession(NS_ERROR_CONNECTION_REFUSED);
+    AbortSession(rv);
-    return NS_ERROR_CONNECTION_REFUSED;
+    return rv;
   }
 
   LOG(("WebSocketChannel::OnStartRequest: HTTP status %d\n", status));