Merge pull request #6209 from mozilla/integration-tests

feat(tests): infrastructure for integration tests

Author: kschelonka

.env

@@ -9,7 +9,7 @@ NEXTAUTH_URL=http://localhost:6060
 DISABLE_DOCKERFLOW=
 
 # Database server
-DATABASE_URL=postgres://postgres@localhost:5432/blurts
+DATABASE_URL=postgres://blurts:blurts@localhost:5432/blurts
 # How many seconds can unverified subscribers remain in the database
 DELETE_UNVERIFIED_SUBSCRIBERS_TIMER=86400
 
@@ -145,9 +145,9 @@ MONTHLY_SCANS_QUOTA=
 STATS_TOKEN=
 
 # GCP PubSub Project ID and subscription name
-GCP_PUBSUB_PROJECT_ID=
-GCP_PUBSUB_TOPIC_NAME=
-GCP_PUBSUB_SUBSCRIPTION_NAME=
+GCP_PUBSUB_PROJECT_ID=your-project-name
+GCP_PUBSUB_TOPIC_NAME=hibp-breaches
+GCP_PUBSUB_SUBSCRIPTION_NAME=hibp-cron
 
 # Randomly-generated UUIDv5 namespace, until/unless we are approved to use FxA UID for Nimbus User ID.
 NIMBUS_UUID_NAMESPACE=00000000-0000-0000-0000-000000000000

.env.ci

@@ -0,0 +1,180 @@
+# local, heroku, stage, production
+APP_ENV=local
+SERVER_URL=http://localhost:6060
+PORT=6060
+NEXTAUTH_URL=http://localhost:6060
+NODE_ENV=development
+
+# 1: disables the dockerflow endpoints
+# see: https://github.com/mozilla-services/Dockerflow#containerized-app-requirements
+DISABLE_DOCKERFLOW=
+
+# Database server
+DATABASE_URL=postgres://blurts:blurts@localhost:5432/test-blurts
+# How many seconds can unverified subscribers remain in the database
+DELETE_UNVERIFIED_SUBSCRIBERS_TIMER=86400
+
+# How many seconds until page tokens expire?
+PAGE_TOKEN_TIMER=0
+
+# Email server
+SMTP_URL=
+# From: address used in emails
+EMAIL_FROM=
+# https://docs.aws.amazon.com/ses/latest/DeveloperGuide/using-configuration-sets.html
+SES_CONFIG_SET=
+# 1: only log messages coming back from SES
+SES_NOTIFICATION_LOG_ONLY=
+
+# s3 bucket for cdn
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_REGION=
+S3_BUCKET=
+
+# Firefox Accounts OAuth
+FXA_SETTINGS_URL=https://accounts.stage.mozaws.net/settings
+
+OAUTH_CLIENT_ID=edd29a80019d61a1
+OAUTH_CLIENT_SECRET=get-this-from-groovecoder-or-fxmonitor-engineering
+OAUTH_AUTHORIZATION_URI=https://accounts.stage.mozaws.net/authorization
+OAUTH_METRICS_FLOW_URI=https://accounts.stage.mozaws.net/metrics-flow
+OAUTH_PROFILE_URI=https://profile.stage.mozaws.net/v1/profile
+OAUTH_TOKEN_URI=https://oauth.stage.mozaws.net/v1/token
+OAUTH_ACCOUNT_URI="https://api-accounts.stage.mozaws.net/v1"
+
+# HIBP API for breach data
+# How many seconds to wait before refreshing upstream breach data from HIBP
+HIBP_RELOAD_BREACHES_TIMER=600
+# HIBP API for range search and subscription
+HIBP_KANON_API_ROOT=https://enterprise.stage-api.haveibeenpwned.com
+HIBP_KANON_API_TOKEN=
+HIBP_API_ROOT=https://haveibeenpwned.com/api/v2
+HIBP_API_TOKEN=
+# How many milliseconds to wait before retrying an HIBP request
+HIBP_THROTTLE_DELAY=2000
+# Max number of times to try an HIBP request before throwing error
+HIBP_THROTTLE_MAX_TRIES=5
+# Authorization token for HIBP to present to /hibp/notify endpoint
+HIBP_NOTIFY_TOKEN=unsafe-default-token-for-dev
+# Domains we prefer to not link to
+HIBP_BREACH_DOMAIN_BLOCKLIST=a-blocked-domain.com,another-blocked-domain.org
+
+# OneRep API for exposure scanning
+ONEREP_API_BASE=https://mozilla.api.onerep.com
+ONEREP_API_KEY=
+ONEREP_WEBHOOK_SECRET="unsafe-default-secret-for-dev"
+
+# Firefox Remote Settings
+FX_REMOTE_SETTINGS_WRITER_SERVER=https://settings-writer.prod.mozaws.net/v1
+FX_REMOTE_SETTINGS_WRITER_USER=
+FX_REMOTE_SETTINGS_WRITER_PASS=
+
+# DSN for Sentry error and event capturing
+# e.g., SENTRY_DSN=https://{key}@sentry.prod.mozaws.net/408
+SENTRY_DSN=
+SENTRY_DSN_LEGACY=
+
+BREACH_RESOLUTION_ENABLED=1
+PRODUCT_PROMOS_ENABLED=1
+
+# Experiment Flag
+EXPERIMENT_ACTIVE=0
+
+REDIS_URL=redis://redis.mock
+
+SUPPORTED_LOCALES=cs,cy,da,de,el,en,en-CA,en-GB,es-AR,es-CL,es-ES,es-MX,fi,fr,fy-NL,gn,hu,kab,ia,id,it,ja,ko,nl,nn-NO,pt-BR,pt-PT,ru,sk,sl,sq,sv-SE,th,tr,uk,vi,zh-CN,zh-TW
+
+# Locales blocked from viewing Mozilla VPN promos. Use CSV without whitespace.
+VPN_PROMO_BLOCKED_LOCALES=zh-CN
+
+# MaxMind GeoLite2 geolocation service used for VPN Banner
+# For Heroku deploys, the following 3 vars are generated automatically via Buildpack https://github.com/HiMamaInc/heroku-buildpack-geoip-geolite2
+# Staging and production environments will need variables set manually
+# Local environment uses a test database with limited data (preset here)
+GEOIP_GEOLITE2_PATH=./tests/mmdb/
+GEOIP_GEOLITE2_CITY_FILENAME=GeoLite2-City-Test.mmdb
+GEOIP_GEOLITE2_COUNTRY_FILENAME=GeoLite2-Country-Test.mmdb
+
+# Educational video src urls, hosted by SRE team on a CDN
+EDUCATION_VIDEO_URL_RELAY=https://monitor.cdn.mozilla.net/videos/FF_Relay_version_02.mp4
+EDUCATION_VIDEO_URL_VPN=https://monitor.cdn.mozilla.net/videos/Mozilla_VPN.mp4
+
+# Email addresses that are allowed to test and send emails
+ADMINS=
+
+# Enable monthly cron-job, currently for sending unresolved breach reminder emails
+MONTHLY_CRON_ENABLED=
+
+# Functional tests
+E2E_TEST_ENV=ci # need to not be 'local' because of imports
+E2E_TEST_SECRET=test-secret
+E2E_TEST_ACCOUNT_BASE_EMAIL=test-account
+E2E_TEST_ACCOUNT_BASE_PASSWORD=test-password
+
+# Monitor Premium features
+# Link to start user on the subscription process. PREMIUM_ENABLED must be set to `true`.
+SUBSCRIPTION_BILLING_AMOUNT_MONTHLY_US=42.42
+SUBSCRIPTION_BILLING_AMOUNT_BUNDLE_INDIVIDUAL_MONTHLY_US=424
+SUBSCRIPTION_BILLING_AMOUNT_BUNDLE_MONTHLY_US=42
+
+# SubPlat 2.0 URL, product and plan IDs, used for Plus subscriptions:
+FXA_SUBSCRIPTIONS_URL=https://accounts.stage.mozaws.net/subscriptions
+PREMIUM_PRODUCT_ID=prod_NErZh679W62lai
+PREMIUM_PLAN_ID_MONTHLY_US=price_1MUNq0Kb9q6OnNsL4BoJgepf
+PREMIUM_PLAN_ID_YEARLY_US=price_1NvqawKb9q6OnNsLRTnYrtrV
+
+# SubPlat 3.0 URL and offering ID, used for Plus subscriptions:
+SUBPLAT_SUBSCRIPTIONS_URL=https://payments-next.stage.fxa.nonprod.webservices.mozgcp.net
+SUBPLAT_MONITOR_OFFERING_ID=monitorplusstage
+SUBPLAT_BUNDLE_OFFERING_ID=privacyprotectionplan/yearly
+SUBPLAT_BUNDLE_PRODUCT_ID=prod_SFb8iVuZIOPREe
+SUBPLAT_BUNDLE_PRICE_ID=price_1RMAopKb9q6OnNsLSGe1vLtt
+
+# Mozilla privacy product URLs
+FIREFOX_RELAY_LANDING_URL=https://stage.fxprivaterelay.nonprod.cloudops.mozgcp.net
+MOZILLA_VPN_LANDING_URL=https://www-dev.allizom.org/products/vpn
+
+# This date is used to direct users who signed up after data broker scanning
+# was released to the welcome flow. Users who had signed up before and thus
+# have seen data breach results before, will be able to see their known breaches
+# first:
+BROKER_SCAN_RELEASE_DATE=2024-02-06
+
+MONTHLY_SUBSCRIBERS_QUOTA=
+MONTHLY_SCANS_QUOTA=
+STATS_TOKEN=
+
+# GCP PubSub Project ID and subscription name
+GCP_PUBSUB_PROJECT_ID=your-project-name
+GCP_PUBSUB_TOPIC_NAME=hibp-breaches
+GCP_PUBSUB_SUBSCRIPTION_NAME=hibp-cron
+PUBSUB_HOST=localhost
+PUBSUB_PORT=8085
+PUBSUB_EMULATOR_HOST="${PUBSUB_HOST}:${PUBSUB_PORT}"
+
+
+# Randomly-generated UUIDv5 namespace, until/unless we are approved to use FxA UID for Nimbus User ID.
+NIMBUS_UUID_NAMESPACE=00000000-0000-0000-0000-000000000000
+NIMBUS_SIDECAR_URL=http://localhost:8001
+
+# The maximum number of jobs that the email breach alert worker will process.
+EMAIL_BREACH_ALERT_MAX_MESSAGES = 10000
+
+# The maximum number of scans and profiles allowed. May be used for alerts, and for redirecting to waitlist.
+MAX_MANUAL_SCANS=100
+MAX_INITIAL_SCANS=100
+MAX_PROFILES_ACTIVATED=100
+MAX_PROFILES_CREATED=100
+
+# Used during CI to upload sourcemaps to Sentry.
+UPLOAD_SENTRY_SOURCEMAPS=false
+SENTRY_AUTH_TOKEN=
+
+# Whether GA4 sends data or not. NOTE: must be set in build environment.
+NEXT_PUBLIC_GA4_DEBUG_MODE=true
+
+GA4_API_SECRET=unsafe-default-secret-for-dev
+
+# Data broker removal estimates data
+DATA_BROKER_REMOVAL_ESTIMATES_DATA=[]

.github/workflows/test_integrations.yml

@@ -0,0 +1,49 @@
+name: "Test Integrations"
+permissions: {}
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test-integrations:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      # Log into Docker Hub to avoid getting rate-limited
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      # Kick off starting docker compose first, since it can take a bit and can run in the background
+      - name: Start Docker Compose services
+        run: docker compose --env-file=./.env.ci up -d
+
+      # While we wait for docker compose to be healthy we install node and needed packages for this service
+      - name: Set up node
+        uses: actions/setup-node@v5
+        with:
+          node-version: 20.19.x
+
+      - name: Install dependencies
+        run: npm ci
+      
+      # Wait for the docker services we started earlier to all be healthy
+      - name: Wait for services to be healthy
+        run: docker compose --env-file=./.env.ci up --wait
+      
+      - name: Set up postgres
+        run: npm run db:migrate
+        env:
+          DATABASE_URL: postgres://blurts:blurts@localhost:5432/test-blurts
+
+      # Let's run those integration tests!
+      - name: Run service integration tests
+        run: npm run test-integrations

README.md

@@ -295,7 +295,7 @@ You can also enforce the alert being sent for a specific email address via the
 # export env var first to persist between runs
 # `export [email protected]`
 HIBP_TEST_EMAIL="[email protected]"; \
-HASH=$(echo -n "$HIBP_TEST_EMAIL" | sha1sum | awk '{print toupper($1)}'); \
+HASH=$(echo -n "$HIBP_TEST_EMAIL" | sha1sum | awk '{print $1}'); \
 PREFIX=${HASH:0:7}; \
 SUFFIX=${HASH:7}; \
 curl -d "{\"breachName\": \"000webhost\", \"hashPrefix\": \"$PREFIX\", \"hashSuffixes\": [\"$SUFFIX\"]}" \

jest.config.cjs

@@ -187,10 +187,7 @@ const customJestConfig = {
   // testLocationInResults: false,
 
   // The glob patterns Jest uses to detect test files
-  // testMatch: [
-  //   "**/__tests__/**/*.[jt]s?(x)",
-  //   "**/?(*.)+(spec|test).[tj]s?(x)"
-  // ],
+  testMatch: ["**/src/**/?(*.)+(spec|test|integration).[tj]s?(x)"],
 
   // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
   testPathIgnorePatterns: ["/node_modules/", "/dist/"],

jest.setup.ts

@@ -55,3 +55,8 @@ jest.mock("./src/envVars", () => {
     getEnvVarsOrThrow: () => process.env,
   };
 });
+
+// Avoiding putting in the env file in case this gets loaded into prod
+// TODO: Centralize and streamline configuration for environments
+// mozilla-hub.atlassian.net/browse/MNTOR-5089
+https: process.env.PUBSUB_EMULATOR_HOST = "localhost:8085";

package.json

@@ -25,7 +25,8 @@
     "start": "next start",
     "lint": "stylelint '**/*.scss' && prettier --check './src' && eslint --max-warnings=0 . && tsc -p tsconfig.json --noEmit && npm run validate-nimbus",
     "fix": "prettier --write './src' && eslint --fix . && stylelint --fix '**/*.scss'",
-    "test": "npm run build-nimbus && jest",
+    "test": "npm run build-nimbus && jest \"\\.(spec|test)\\.ts\"",
+    "test-integrations": "npm run build-nimbus && jest \"\\.integration\\.ts\" --coverage=false --runInBand",
     "functional-tests": "playwright test functional-tests/ --config=functional-tests/playwright.config.ts",
     "functional-tests:debug": "playwright test functional-tests/ --config=functional-tests/playwright.config.ts --ui",
     "cron:first-data-broker-removal-fixed": "node dist/scripts/cronjobs/firstDataBrokerRemovalFixed.js",

src/db/knexfile.js

@@ -43,7 +43,7 @@ const TESTS_CONFIG = {
   connection: testConnectionObj,
 };
 
-let exportConfig = NODE_ENV === "tests" ? TESTS_CONFIG : RUNTIME_CONFIG;
+let exportConfig = NODE_ENV === "test" ? TESTS_CONFIG : RUNTIME_CONFIG;
 
 if (APP_ENV === "cloudrun") {
   // @ts-ignore TODO: Check if this typing error is correct, or if the types are wrong?

src/scripts/cronjobs/emailBreachAlerts.integration.ts

@@ -0,0 +1,27 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { Knex } from "knex";
+import createDbConnection from "../../db/connect";
+import { PubSub } from "@google-cloud/pubsub";
+
+describe("Email breach functional tests", () => {
+  let conn: Knex;
+  beforeAll(() => {
+    conn = createDbConnection();
+  });
+  afterAll(async () => {
+    await conn.destroy();
+  });
+  it("connects to the database", async () => {
+    const res = await conn.raw(`select 1 as connected`);
+    expect(res.rows).toStrictEqual([{ connected: 1 }]);
+  });
+  it("connects to pubsub", () => {
+    const projectId = process.env.GCP_PUBSUB_PROJECT_ID;
+    const subscriptionName = process.env.GCP_PUBSUB_SUBSCRIPTION_NAME!;
+    const pubsub = new PubSub({ projectId });
+    expect(() => pubsub.subscription(subscriptionName)).not.toThrow();
+  });
+});

src/scripts/cronjobs/emailBreachAlerts.tsx

@@ -383,7 +383,12 @@ async function getDataSummary(
 /* c8 ignore start */
 function createPubSubClient() {
   let options = {};
-  if (process.env.NODE_ENV === "development") {
+  // TODO - Consolidate configuration logic for this and other clients
+  // https://mozilla-hub.atlassian.net/browse/MNTOR-5089
+  if (
+    process.env.NODE_ENV === "development" ||
+    process.env.NODE_ENV === "test"
+  ) {
     console.debug("Dev mode, connecting to local pubsub emulator");
     options = {
       servicePath: "localhost",