Fixing PSS encoding bug

Author: invisal

src/rsa/emsa_pss.ts

@@ -30,7 +30,7 @@ export function emsa_pss_encode(
   const maskedDB = xor(db, dbMask);
 
   const leftMost = 8 * emLen - emBits;
-  maskedDB[0] = maskedDB[0] && (0xff >> leftMost);
+  maskedDB[0] = maskedDB[0] & (0xff >> leftMost);
 
   return new Uint8Array([...maskedDB, ...h, 0xbc]);
 }
@@ -57,7 +57,7 @@ export function emsa_pss_verify(
 
   const dbMask = mgf1(h, emLen - hLen - 1, algorithm);
   const db = xor(maskedDB, dbMask);
-  db[0] = db[0] && (0xff >> leftMost);
+  db[0] = db[0] & (0xff >> leftMost);
 
   for (let i = 1; i < emLen - hLen - sLen - 2; i++) {
     if (db[i] !== 0x00) return false;

src/rsa/rsa_internal.ts

@@ -138,6 +138,7 @@ export function rsa_pkcs1_sign(
   n: bigint,
   d: bigint,
   message: Uint8Array,
+  algorithm: RSAHashAlgorithm,
 ): RawBinary {
   // deno-fmt-ignore
   const oid = [
@@ -153,10 +154,11 @@ export function rsa_pkcs1_sign(
     0x03,
     0x04,
     0x02,
-    0x01,
+    algorithm === "sha512" ? 0x03 : 0x01, // <--
     0x05,
     0x00,
   ];
+
   const der = [
     0x30,
     message.length + 2 + oid.length,

src/rsa/rsa_js.ts

@@ -69,6 +69,7 @@ export class PureRSA {
         key.n,
         key.d,
         digest(options.hash, message),
+        options.hash,
       );
     } else {
       return rsassa_pss_sign(key, message, options.hash);