use OpenPGP.js from web content script, simplify

.semaphore/semaphore.yml

@@ -37,6 +37,9 @@ blocks:
         - name: consumer mock - flaky test group
           commands:
             - npm run-script test_ci_chrome_consumer_flaky
+        - name: content script tests
+          commands:
+            - npm run-script test_ci_chrome_content_scripts
         - name: enterprise mock - standard test group
           commands:
             - npm run-script test_ci_chrome_enterprise

.vscode/launch.json

@@ -21,6 +21,16 @@
       "outputCapture": "std",
       "skipFiles": ["<node_internals>/**/*.js"]
     },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug buf",
+      "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/ava",
+      "preLaunchTask": "npm: pretest-incremental",
+      "runtimeArgs": ["build/test/test/source/buf.js", "--verbose", "--concurrency=1"],
+      "outputCapture": "std",
+      "skipFiles": ["<node_internals>/**/*.js"]
+    },
     {
       "type": "node",
       "request": "launch",
@@ -81,6 +91,25 @@
       "outputCapture": "std",
       "skipFiles": ["<node_internals>/**/*.js"]
     },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug chrome_content_scripts",
+      "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/ava",
+      "preLaunchTask": "npm: pretest-incremental",
+      "runtimeArgs": [
+        "build/test/test/source/test.js",
+        "--verbose",
+        "--concurrency=1",
+        "--",
+        "CONTENT-SCRIPT-TESTS",
+        "--retry=false",
+        "--debug",
+        "--pool-size=1"
+      ],
+      "outputCapture": "std",
+      "skipFiles": ["<node_internals>/**/*.js"]
+    },
     {
       "type": "node",
       "request": "launch",
@@ -120,6 +149,39 @@
       ],
       "outputCapture": "std",
       "skipFiles": ["<node_internals>/**/*.js"]
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug tsc-compiler",
+      "program": "${workspaceFolder}/build/tooling/tsc-compiler.js",
+      "cwd": "${workspaceFolder}",
+      "args": ["--project", "conf/tsconfig.streams.json"],
+      "stopOnEntry": true,
+      "outputCapture": "std",
+      "skipFiles": ["<node_internals>/**/*.js"]
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug resolve-modules",
+      "program": "${workspaceFolder}/build/tooling/resolve-modules.js",
+      "cwd": "${workspaceFolder}",
+      "args": ["--project", "./tsconfig.json"],
+      "stopOnEntry": true,
+      "outputCapture": "std",
+      "skipFiles": ["<node_internals>/**/*.js"]
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug bundle-content-scripts",
+      "program": "${workspaceFolder}/build/tooling/bundle-content-scripts.js",
+      "cwd": "${workspaceFolder}",
+      "args": [],
+      "stopOnEntry": true,
+      "outputCapture": "std",
+      "skipFiles": ["<node_internals>/**/*.js"]
     }
   ]
 }

conf/webpack.config.js

@@ -0,0 +1,24 @@
+//webpack.config.js
+//bundle the web version of @openpgp/web-stream-tools for content script
+const path = require('path');
+
+module.exports = {
+  mode: 'production',
+  entry: {
+    main: '../build/generic-extension-wip/lib/streams/streams.js',
+  },
+  output: {
+    library: {
+      name: 'Stream',
+      type: 'var',
+    },
+    path: path.resolve('../build/generic-extension-wip/lib'),
+    filename: 'streams_web.js', // <--- Will be compiled to this single file
+  },
+  resolve: {
+    fallback: {
+      stream: false,
+    },
+    extensions: ['.js'],
+  },
+};

extension/js/background_page/background_page.ts

@@ -7,7 +7,6 @@ import { Bm, BrowserMsg } from '../common/browser/browser-msg.js';
 import { emailKeyIndex } from '../common/core/common.js';
 import { VERSION } from '../common/core/const.js';
 import { ExpirationCache } from '../common/core/expiration-cache.js';
-import { processAndStoreKeysFromEkmLocally, getLocalKeyExpiration } from '../common/helpers.js';
 import { Catch } from '../common/platform/catch.js';
 import { AcctStore } from '../common/platform/store/acct-store.js';
 import { ContactStore } from '../common/platform/store/contact-store.js';
@@ -60,8 +59,6 @@ console.info('background_process.js starting');
   BrowserMsg.bgAddListener('storeGlobalSet', (r: Bm.StoreGlobalSet) => GlobalStore.set(r.values));
   BrowserMsg.bgAddListener('storeAcctGet', (r: Bm.StoreAcctGet) => AcctStore.get(r.acctEmail, r.keys));
   BrowserMsg.bgAddListener('storeAcctSet', (r: Bm.StoreAcctSet) => AcctStore.set(r.acctEmail, r.values));
-  BrowserMsg.bgAddListener('processAndStoreKeysFromEkmLocally', processAndStoreKeysFromEkmLocally);
-  BrowserMsg.bgAddListener('getLocalKeyExpiration', getLocalKeyExpiration);
 
   // todo - when https://github.com/FlowCrypt/flowcrypt-browser/issues/2560
   //   is fixed, this can be moved to the gmail content script, and some may be removed

extension/js/common/browser/browser-msg.ts

@@ -83,8 +83,6 @@ export namespace Bm {
   export type ShowAttachmentPreview = { iframeUrl: string };
   export type ReRenderRecipient = { email: string };
   export type SaveFetchedPubkeys = { email: string; pubkeys: string[] };
-  export type ProcessAndStoreKeysFromEkmLocally = { acctEmail: string; decryptedPrivateKeys: string[] };
-  export type GetLocalKeyExpiration = { acctEmail: string };
 
   export namespace Res {
     export type GetActiveTabInfo = {
@@ -107,12 +105,6 @@ export namespace Bm {
     export type AjaxGmailAttachmentGetChunk = { chunk: Buf };
     export type _tab_ = { tabId: string | null | undefined }; // eslint-disable-line @typescript-eslint/naming-convention
     export type SaveFetchedPubkeys = boolean;
-    export type GetLocalKeyExpiration = number | undefined;
-    export type ProcessAndStoreKeysFromEkmLocally = {
-      needPassphrase?: boolean;
-      updateCount?: number;
-      noKeysSetup?: boolean;
-    };
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     export type Db = any; // not included in Any below
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -134,9 +126,7 @@ export namespace Bm {
       | StoreGlobalSet
       | AjaxGmailAttachmentGetChunk
       | SaveFetchedPubkeys
-      | ProcessAndStoreKeysFromEkmLocally
-      | PgpKeyBinaryToArmored
-      | GetLocalKeyExpiration;
+      | PgpKeyBinaryToArmored;
   }
 
   export type AnyRequest =
@@ -176,8 +166,6 @@ export namespace Bm {
     | ShowAttachmentPreview
     | ReRenderRecipient
     | SaveFetchedPubkeys
-    | ProcessAndStoreKeysFromEkmLocally
-    | GetLocalKeyExpiration
     | PgpKeyBinaryToArmored
     | AuthWindowResult;
 
@@ -244,10 +232,6 @@ export class BrowserMsg {
           BrowserMsg.sendAwait(undefined, 'pgpKeyBinaryToArmored', bm, true) as Promise<Bm.Res.PgpKeyBinaryToArmored>,
         saveFetchedPubkeys: (bm: Bm.SaveFetchedPubkeys) =>
           BrowserMsg.sendAwait(undefined, 'saveFetchedPubkeys', bm, true) as Promise<Bm.Res.SaveFetchedPubkeys>,
-        processAndStoreKeysFromEkmLocally: (bm: Bm.ProcessAndStoreKeysFromEkmLocally) =>
-          BrowserMsg.sendAwait(undefined, 'processAndStoreKeysFromEkmLocally', bm, true) as Promise<Bm.Res.ProcessAndStoreKeysFromEkmLocally>,
-        getLocalKeyExpiration: (bm: Bm.GetLocalKeyExpiration) =>
-          BrowserMsg.sendAwait(undefined, 'getLocalKeyExpiration', bm, true) as Promise<Bm.Res.GetLocalKeyExpiration>,
       },
     },
     passphraseEntry: (dest: Bm.Dest, bm: Bm.PassphraseEntry) => BrowserMsg.sendCatch(dest, 'passphrase_entry', bm),

extension/js/common/helpers.ts

@@ -10,7 +10,6 @@ import { ClientConfiguration } from './client-configuration.js';
 import { ContactStore } from './platform/store/contact-store.js';
 import { KeyStore } from './platform/store/key-store.js';
 import { PassphraseStore } from './platform/store/passphrase-store.js';
-import { Bm } from './browser/browser-msg.js';
 import { PgpPwd } from './core/crypto/pgp/pgp-password.js';
 
 export const isCustomerUrlFesUsed = async (acctEmail: string) => {
@@ -111,9 +110,11 @@ export const processAndStoreKeysFromEkmLocally = async ({
   acctEmail,
   decryptedPrivateKeys,
   ppOptions: originalOptions,
-}: Bm.ProcessAndStoreKeysFromEkmLocally & {
+}: {
+  acctEmail: string;
+  decryptedPrivateKeys: string[];
   ppOptions?: PassphraseOptions;
-}): Promise<Bm.Res.ProcessAndStoreKeysFromEkmLocally> => {
+}) => {
   const { unencryptedPrvs } = await parseAndCheckPrivateKeys(decryptedPrivateKeys);
   const existingKeys = await KeyStore.get(acctEmail);
   let { keysToRetain, newUnencryptedKeysToSave } = await filterKeysToSave(unencryptedPrvs, existingKeys);
@@ -175,7 +176,7 @@ export const processAndStoreKeysFromEkmLocally = async ({
   };
 };
 
-export const getLocalKeyExpiration = async ({ acctEmail }: Bm.GetLocalKeyExpiration): Promise<Bm.Res.GetLocalKeyExpiration> => {
+export const getLocalKeyExpiration = async (acctEmail: string): Promise<number> => {
   const kis = await KeyStore.get(acctEmail);
   const expirations = await Promise.all(kis.map(async ki => (await KeyUtil.parse(ki.public))?.expiration ?? Number.MAX_SAFE_INTEGER));
   return Math.max(...expirations);

extension/js/common/platform/require.ts

@@ -23,6 +23,7 @@
 
 import { MimeParser } from '../core/types/emailjs.js';
 import type * as OpenPGP from 'openpgp';
+import { Catch } from './catch.js';
 
 type Codec = {
   encode: (text: string, mode: 'fatal' | 'html') => string;
@@ -32,7 +33,16 @@ type Codec = {
 };
 
 export const requireOpenpgp = (): typeof OpenPGP => {
-  return (window as unknown as { openpgp: typeof OpenPGP }).openpgp;
+  if (window !== globalThis && Catch.browser().name === 'firefox') {
+    // fix Firefox sandbox permission issues as per convo https://github.com/FlowCrypt/flowcrypt-browser/pull/5013#discussion_r1148343995
+    window.Uint8Array.prototype.subarray = function (...args) {
+      return new Uint8Array(this).subarray(...args);
+    };
+    window.Uint8Array.prototype.slice = function (...args) {
+      return new Uint8Array(this).slice(...args);
+    };
+  }
+  return (globalThis as unknown as { openpgp: typeof OpenPGP }).openpgp;
 };
 
 export const requireMimeParser = (): typeof MimeParser => {

extension/js/content_scripts/webmail/gmail-element-replacer.ts

@@ -16,7 +16,6 @@ import { Gmail } from '../../common/api/email-provider/gmail/gmail.js';
 import { Injector } from '../../common/inject.js';
 import { PubLookup } from '../../common/api/pub-lookup.js';
 import { Notifications } from '../../common/notifications.js';
-// note: only types are supported for OpenPGP.js or web-stream-tools, their function calls will fail
 import { PgpArmor } from '../../common/core/crypto/pgp/pgp-armor.js';
 import { Ui } from '../../common/browser/ui.js';
 import { WebmailCommon } from '../../common/webmail.js';
@@ -391,7 +390,7 @@ export class GmailElementReplacer implements WebmailElementReplacer {
           if (this.debug) {
             console.debug('processNewPgpAttachments() -> msgGet may take some time');
           }
-          const msg = await this.gmail.msgGet(msgId, 'full');
+          const msg = await this.gmail.msgGet(msgId, 'full'); // todo: cache or thoroughly refactor in #5022
           if (this.debug) {
             console.debug('processNewPgpAttachments() -> msgGet done -> processAttachments', msg);
           }

extension/js/content_scripts/webmail/setup-webmail-content-script.ts

@@ -14,6 +14,7 @@ import { Ui } from '../../common/browser/ui.js';
 import { ClientConfiguration, ClientConfigurationError } from '../../common/client-configuration.js';
 import { Str, Url } from '../../common/core/common.js';
 import { InMemoryStoreKeys, VERSION } from '../../common/core/const.js';
+import { getLocalKeyExpiration, processAndStoreKeysFromEkmLocally } from '../../common/helpers.js';
 import { Injector } from '../../common/inject.js';
 import { Lang } from '../../common/lang.js';
 import { Notifications } from '../../common/notifications.js';
@@ -291,7 +292,7 @@ export const contentScriptSetupIfVacant = async (webmailSpecific: WebmailSpecifi
     ppEvent: { entered?: boolean }
   ) => {
     try {
-      const { needPassphrase, updateCount, noKeysSetup } = await BrowserMsg.send.bg.await.processAndStoreKeysFromEkmLocally({
+      const { needPassphrase, updateCount, noKeysSetup } = await processAndStoreKeysFromEkmLocally({
         acctEmail,
         decryptedPrivateKeys,
       });
@@ -382,7 +383,7 @@ export const contentScriptSetupIfVacant = async (webmailSpecific: WebmailSpecifi
   };
 
   const notifyExpiringKeys = async (acctEmail: string, clientConfiguration: ClientConfiguration, notifications: Notifications) => {
-    const expiration = await BrowserMsg.send.bg.await.getLocalKeyExpiration({ acctEmail });
+    const expiration = await getLocalKeyExpiration(acctEmail);
     if (expiration === undefined) {
       return;
     }

extension/manifest.json

@@ -40,27 +40,13 @@
   ],
   "content_scripts": [
     {
-      "matches": [
-        "https://mail.google.com/*"
-      ],
-      "css": [
-        "/css/webmail.css",
-        "/css/sweetalert2.css"
-      ],
-      "js": [
-        "/lib/purify.js",
-        "/lib/jquery.min.js",
-        "/lib/sweetalert2.js",
-        "/js/content_scripts/webmail_bundle.js"
-      ]
+      "matches": ["https://mail.google.com/*"],
+      "css": ["/css/webmail.css", "/css/sweetalert2.css"],
+      "js": ["/lib/purify.js", "/lib/jquery.min.js", "/lib/openpgp.js", "/lib/sweetalert2.js", "/lib/streams_web.js", "/js/content_scripts/webmail_bundle.js"]
     },
     {
-      "matches": [
-        "https://www.google.com/robots.txt*"
-      ],
-      "js": [
-        "/js/common/oauth2/oauth2_inject.js"
-      ],
+      "matches": ["https://www.google.com/robots.txt*"],
+      "js": ["/js/common/oauth2/oauth2_inject.js"],
       "run_at": "document_start"
     }
   ],
@@ -93,4 +79,4 @@
   ],
   "minimum_chrome_version": "96",
   "content_security_policy": "upgrade-insecure-requests; script-src 'self'; frame-ancestors https://mail.google.com 'self'; img-src 'self' https://* data: blob:; frame-src 'self' blob:; worker-src 'self'; form-action 'none'; media-src 'none'; font-src 'none'; manifest-src 'none'; object-src 'none'; base-uri 'self'"
-}
+}