Bug 1895277 - Update nsAuthSSPI to check certificate hash algorithm a=dmeehan DONTBUILD

Original Revision: https://phabricator.services.mozilla.com/D262779

Differential Revision: https://phabricator.services.mozilla.com/D265280

extensions/auth/nsAuthSSPI.cpp

@@ -23,9 +23,14 @@
 #include "nsCOMPtr.h"
 #include "nsICryptoHash.h"
 #include "mozilla/glean/SecurityManagerSslMetrics.h"
+#include "mozilla/StaticPrefs_network.h"
 
 #include <windows.h>
 
+// for safer certificate parsing
+#include "nss/mozpkix/pkixutil.h"
+#include "nss/mozpkix/pkixder.h"
+
 #define SEC_SUCCESS(Status) ((Status) >= 0)
 
 #ifndef KERB_WRAP_NO_ENCRYPT
@@ -276,8 +281,6 @@ nsAuthSSPI::GetNextToken(const void* inToken, uint32_t inTokenLen,
   // String for end-point bindings.
   const char end_point[] = "tls-server-end-point:";
   const int end_point_length = sizeof(end_point) - 1;
-  const int hash_size = 32;  // Size of a SHA256 hash.
-  const int cbt_size = hash_size + end_point_length;
 
   SECURITY_STATUS rc;
   MS_TimeStamp ignored;
@@ -332,7 +335,63 @@ nsAuthSSPI::GetNextToken(const void* inToken, uint32_t inTokenLen,
       // If we have stored a certificate, the Channel Binding Token
       // needs to be generated and sent in the first input buffer.
       if (mCertDERLength > 0) {
-        // First we create a proper Endpoint Binding structure.
+        // Default to SHA256 for compatibility, but detect SHA384 and SHA512
+        uint32_t hashAlgorithm = nsICryptoHash::SHA256;
+        uint32_t hashSize = 32;  // SHA256 hash size
+
+        // Compute the hash size.
+        [&]() {
+          if (!mozilla::StaticPrefs::network_auth_sspi_detect_hash()) {
+            // This check only exists to make sure that the hash algorithm check
+            // doesn't break previous working behaviour.
+            return;
+          }
+          using namespace mozilla::pkix;
+          Input certDER;
+
+          mozilla::pkix::Result pkixResult = certDER.Init(
+              static_cast<const uint8_t*>(mCertDERData), mCertDERLength);
+          if (pkixResult != Success) {
+            return;
+          }
+
+          BackCert cert(certDER, EndEntityOrCA::MustBeEndEntity, nullptr);
+          pkixResult = cert.Init();
+          if (pkixResult != Success) {
+            return;
+          }
+
+          // Parse the signature algorithm from the signed data
+          der::PublicKeyAlgorithm publicKeyAlg;
+          DigestAlgorithm digestAlg;
+          Reader signatureAlgorithmReader(cert.GetSignedData().algorithm);
+          pkixResult = der::SignatureAlgorithmIdentifierValue(
+              signatureAlgorithmReader, publicKeyAlg, digestAlg);
+
+          if (pkixResult != Success) {
+            return;
+          }
+          // Map digest algorithms to hash algorithms for Extended Protection
+          switch (digestAlg) {
+            case DigestAlgorithm::sha384:
+              hashAlgorithm = nsICryptoHash::SHA384;
+              hashSize = 48;  // SHA384 hash size
+              break;
+            case DigestAlgorithm::sha512:
+              hashAlgorithm = nsICryptoHash::SHA512;
+              hashSize = 64;  // SHA512 hash size
+              break;
+            case DigestAlgorithm::sha256:
+            default:
+              // Use SHA256 as default for compatibility
+              hashAlgorithm = nsICryptoHash::SHA256;
+              hashSize = 32;
+              break;
+          }
+        }();
+
+        // Create Endpoint Binding structure with correct size
+        const int cbt_size = hashSize + end_point_length;
         pendpoint_binding.dwInitiatorAddrType = 0;
         pendpoint_binding.cbInitiatorLength = 0;
         pendpoint_binding.dwInitiatorOffset = 0;
@@ -363,17 +422,19 @@ nsAuthSSPI::GetNextToken(const void* inToken, uint32_t inTokenLen,
         memcpy(sspi_cbt_ptr, end_point, end_point_length);
         sspi_cbt_ptr += end_point_length;
 
-        // Start hashing. We are always doing SHA256, but depending
-        // on the certificate, a different alogirthm might be needed.
         nsAutoCString hashString;
+        nsresult rv = NS_ERROR_FAILURE;
 
-        nsresult rv;
         nsCOMPtr<nsICryptoHash> crypto;
         crypto = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
-        if (NS_SUCCEEDED(rv)) rv = crypto->Init(nsICryptoHash::SHA256);
-        if (NS_SUCCEEDED(rv))
+        if (NS_SUCCEEDED(rv)) {
+          rv = crypto->Init(hashAlgorithm);
+        }
+        if (NS_SUCCEEDED(rv)) {
           rv = crypto->Update((unsigned char*)mCertDERData, mCertDERLength);
+        }
         if (NS_SUCCEEDED(rv)) rv = crypto->Finish(false, hashString);
+
         if (NS_FAILED(rv)) {
           free(mCertDERData);
           mCertDERData = nullptr;
@@ -382,10 +443,9 @@ nsAuthSSPI::GetNextToken(const void* inToken, uint32_t inTokenLen,
           return rv;
         }
 
-        // Once the hash has been computed, we store it in memory right
-        // after the Endpoint structure and the "tls-server-end-point:"
-        // char array.
-        memcpy(sspi_cbt_ptr, hashString.get(), hash_size);
+        // Store the computed hash in memory right after the Endpoint
+        // structure and the "tls-server-end-point:" char array
+        memcpy(sspi_cbt_ptr, hashString.get(), hashSize);
 
         // Free memory used to store the server certificate
         free(mCertDERData);

modules/libpref/init/StaticPrefList.yaml

@@ -13017,6 +13017,13 @@
   value: false
   mirror: always
 
+# When true, the hashing of the certificate will be done with the hashing
+# algorithm specified in the cert instead of SHA256.
+- name: network.auth.sspi_detect_hash
+  type: RelaxedAtomicBool
+  value: true
+  mirror: always
+
 # See the full list of values in nsICookieService.idl.
 - name: network.cookie.cookieBehavior
   type: RelaxedAtomicInt32