fix: tls serial number causes illegal padding error (#2459)

This is a hack to work around PeculiarVentures/x509#74
until it is addressed upstream.

It seems serial numbers starting with `80` cause `@peculiar/x509` to
generate invalid certifiates that Node's `TLSSocket` then fails to
parse, throwing an `ERR_OSSL_ASN1_ILLEGAL_PADDING` error, so the hack
is to generate serial numbers until we get one that doesn't start with
`80`.

This can be reverted when the upstream issue is fixed.

Author: achingbrain

packages/connection-encrypter-tls/src/utils.ts

@@ -153,8 +153,7 @@ export async function generateCertificate (peerId: PeerId): Promise<{ cert: stri
   }
 
   const selfCert = await x509.X509CertificateGenerator.createSelfSigned({
-    serialNumber: uint8ArrayToString(crypto.getRandomValues(new Uint8Array(9)), 'base16'),
-    name: '',
+    serialNumber: generateSerialNumber(),
     notBefore: new Date(now - CERT_VALIDITY_PERIOD_FROM),
     notAfter: new Date(now + CERT_VALIDITY_PERIOD_TO),
     signingAlgorithm: alg,
@@ -186,6 +185,19 @@ export async function generateCertificate (peerId: PeerId): Promise<{ cert: stri
   }
 }
 
+function generateSerialNumber (): string {
+  // HACK: serial numbers starting with 80 generated by @peculiar/x509 don't
+  // work with TLSSocket, remove when https://github.com/PeculiarVentures/x509/issues/74
+  // is resolved
+  while (true) {
+    const serialNumber = (Math.random() * Math.pow(2, 52)).toFixed(0)
+
+    if (!serialNumber.startsWith('80')) {
+      return serialNumber
+    }
+  }
+}
+
 /**
  * @see https://github.com/libp2p/specs/blob/master/tls/tls.md#libp2p-public-key-extension
  */