Bug 1828968 - osclientcerts: make RSA-PSS support configurable via pr…

…ef r=jschanck

Due to design constraints, it is difficult for osclientcerts to properly
indicate whether or not each known key supports RSA-PSS. Ideally such a
determination would be made close to when a particular key is going to be used,
but due to the design of PKCS#11 and NSS' tight coupling to it, osclientcerts
would have to make this determination when searching for all known keys, which
has been shown to be prohibitively slow on Windows and results in unexpected
dialogs on macOS.

Thus, previously osclientcerts simply assumed all RSA keys supported RSA-PSS.
This has resulted in handshake failures when a server indicates that it accepts
RSA-PSS signatures.

This patch instead makes RSA-PSS support configurable via a pref
(security.osclientcerts.assume_rsa_pss_support). If the pref is true,
osclientcerts assumes all RSA keys support RSA-PSS. If it is false, it assumes
no RSA keys support RSA-PSS.

Differential Revision: https://phabricator.services.mozilla.com/D175966

browser/components/enterprisepolicies/Policies.sys.mjs

@@ -1731,6 +1731,7 @@ export var Policies = {
         "security.insecure_connection_text.enabled",
         "security.insecure_connection_text.pbmode.enabled",
         "security.mixed_content.block_active_content",
+        "security.osclientcerts.assume_rsa_pss_support",
         "security.osclientcerts.autoload",
         "security.OCSP.enabled",
         "security.OCSP.require",

modules/libpref/init/StaticPrefList.yaml

@@ -13855,6 +13855,17 @@
   value: true
   mirror: always
 
+# If true, assume tokens accessed via osclientcerts implement RSA-PSS. If a
+# given token does not support RSA-PSS, users may see the error
+# 'SEC_ERROR_PKCS11_GENERAL_ERROR' if a server indicates it will accept an
+# RSA-PSS signature in the client's certificate verify message.
+# Setting this to false may allow such connections to succeed, if the server
+# also accepts RSA-PKCS1 signatures.
+- name: security.osclientcerts.assume_rsa_pss_support
+  type: RelaxedAtomicBool
+  value: true
+  mirror: always
+
 - name: security.pki.cert_short_lifetime_in_days
   type: RelaxedAtomicUint32
   value: 10

security/certverifier/NSSCertDBTrustDomain.cpp

@@ -24,6 +24,7 @@
 #include "mozilla/Logging.h"
 #include "mozilla/PodOperations.h"
 #include "mozilla/Services.h"
+#include "mozilla/StaticPrefs_security.h"
 #include "mozilla/SyncRunnable.h"
 #include "mozilla/TimeStamp.h"
 #include "mozilla/Unused.h"
@@ -1801,8 +1802,12 @@ bool LoadOSClientCertsModule(const nsCString& dir) {
     return false;
   }
 #endif
+  nsLiteralCString params =
+      StaticPrefs::security_osclientcerts_assume_rsa_pss_support()
+          ? "RSA-PSS"_ns
+          : ""_ns;
   return LoadUserModuleAt(kOSClientCertsModuleName, "osclientcerts", dir,
-                          nullptr);
+                          params.get());
 }
 
 bool LoadLoadableRoots(const nsCString& dir) {