Support for displaying "preview" bucket data from RS.

Trikolon · Trikolon · commit 4ace6c25f180 · 2025-06-30T22:52:19.000+02:00
"preview" is for changes that are still pending review.

This patch also reefactors how the Remote Settings urls are built and consolidates the logic between the NodeJS script and the main web app.
diff --git a/package.json b/package.json
@@ -9,7 +9,7 @@
     "dev": "vite",
     "build": "tsc && vite build",
     "preview": "vite preview",
-    "update-types": "node scripts/updateTypesFromSchema.cjs"
+    "update-types": "node scripts/updateTypesFromSchema.mjs"
   },
   "author": "Emma Zuehlcke",
   "license": "MPL-2.0",
diff --git a/scripts/rs-config.d.ts b/scripts/rs-config.d.ts
@@ -0,0 +1,11 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import type { RS_BASE_URLS } from "./rs-config.js";
+
+export type RSEnvironment = "dev" | "stage" | "prod";
+
+export function isRSEnvValid(env: string | null): env is RSEnvironment;
+export function getRSEndpoint(env: RSEnvironment, isPreview: boolean): URL;
+export function getRSEndpointMeta(env: RSEnvironment, isPreview: boolean): URL;
diff --git a/scripts/rs-config.js b/scripts/rs-config.js
@@ -0,0 +1,64 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Base URLs for different Remote Settings environments
+export const RS_BASE_URLS = {
+  prod: "https://firefox.settings.services.mozilla.com",
+  stage: "https://firefox.settings.services.allizom.org",
+  dev: "https://remote-settings-dev.allizom.org",
+};
+
+// Collection name
+const RS_COLLECTION = "url-classifier-exceptions";
+
+export function isRSEnvValid(env) {
+  return env && Object.keys(RS_BASE_URLS).includes(env);
+}
+
+/**
+ * Build a Remote Settings URL for a given environment and bucket
+ * @param {string} baseUrl - The base URL for the Remote Settings environment.
+ * @param {string} bucket - The bucket name.
+ * @param {string} collection - The collection name.
+ * @param {string} suffix - The suffix to add to the URL.
+ * @returns {URL} The Remote Settings URL.
+ */
+function buildRSUrl(baseUrl, bucket, collection, suffix) {
+  if (!baseUrl || !bucket || !collection) {
+    throw new Error("Invalid parameters");
+  }
+
+  let urlStr = `${baseUrl}/v1/buckets/${bucket}/collections/${collection}`;
+
+  if (suffix) {
+    urlStr += `/${suffix}`;
+  }
+
+  return new URL(urlStr);
+}
+
+/**
+ * Get the Remote Settings endpoint for records.
+ * @param {string} env - The environment to use.
+ * @param {boolean} isPreview - Whether to use the preview bucket.
+ * @returns {URL} The Remote Settings URL.
+ */
+export function getRSEndpoint(env, isPreview) {
+  return buildRSUrl(
+    RS_BASE_URLS[env],
+    isPreview ? "main-preview" : "main",
+    RS_COLLECTION,
+    "records",
+  );
+}
+
+/**
+ * Get the Remote Settings endpoint for collection metadata.
+ * @param {string} env - The environment to use.
+ * @param {boolean} isPreview - Whether to use the preview bucket.
+ * @returns {URL} The Remote Settings URL.
+ */
+export function getRSEndpointMeta(env, isPreview) {
+  return buildRSUrl(RS_BASE_URLS[env], isPreview ? "main-preview" : "main", RS_COLLECTION);
+}
diff --git a/scripts/updateTypesFromSchema.mjs b/scripts/updateTypesFromSchema.mjs
@@ -2,36 +2,31 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-const fs = require("fs");
-const path = require("path");
-const { compile } = require("json-schema-to-typescript");
-require("dotenv").config();
-
-const RS_ENDPOINTS = {
-  prod: "https://firefox.settings.services.mozilla.com",
-  stage: "https://firefox.settings.services.allizom.org",
-  dev: "https://remote-settings-dev.allizom.org",
-};
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { compile } from "json-schema-to-typescript";
+import { getRSEndpointMeta } from "./rs-config.js";
+import "dotenv/config";
 
 /**
  * Get the Remote Settings endpoint to use.
- * Can be overridden by setting VITE_RS_ENVIRONMENT in .env to one of: "prod", "stage", "dev"
+ * Can be overridden by setting VITE_RS_ENVIRONMENT in .env to one of: "prod",
+ * "prod_preview", "stage", "stage_preview", "dev", "dev_preview".
  * Defaults to "prod" if not set or invalid
  */
-function getRSEndpoint() {
-  const env = process.env.VITE_RS_ENVIRONMENT;
-  return RS_ENDPOINTS[env] || RS_ENDPOINTS.prod;
+function getRSUrl() {
+  const env = process.env.VITE_RS_ENVIRONMENT || "prod";
+  return getRSEndpointMeta(env, false);
 }
 
 async function generateTypes() {
   try {
-    // Fetch the schema from Remote Settings
-    const rsOrigin = getRSEndpoint();
-    console.info(`Fetching schema from ${rsOrigin}`);
+    // Fetch the schema from Remote Settings collection metadata
+    const rsUrl = getRSUrl();
+    console.info(`Fetching schema from ${rsUrl.toString()}`);
 
-    const response = await fetch(
-      `${rsOrigin}/v1/buckets/main/collections/url-classifier-exceptions`,
-    );
+    const response = await fetch(rsUrl.toString());
     const data = await response.json();
     const schema = data.data.schema;
 
@@ -44,6 +39,8 @@ async function generateTypes() {
       },
     });
 
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = path.dirname(__filename);
     const srcDir = path.join(__dirname, "..", "src");
 
     // Write the types to a file
diff --git a/src/app.ts b/src/app.ts
@@ -5,6 +5,7 @@
 import { LitElement, html, css } from "lit";
 import { customElement, state } from "lit/decorators.js";
 import { ExceptionListEntry, BugMetaMap } from "./types";
+import { getRSEndpoint, RSEnvironment, isRSEnvValid } from "../scripts/rs-config.js";
 import "./exceptions-table/exceptions-table";
 import "./exceptions-table/top-exceptions-table";
 import "./github-corner";
@@ -13,51 +14,52 @@ const GITHUB_URL = "https://github.com/mozilla/url-classifier-exceptions-ui";
 
 // Query parameter which can be used to override the RS environment.
 const QUERY_PARAM_RS_ENV = "rs_env";
-
-// The available Remote Settings endpoints.
-const RS_ENDPOINTS = {
-  prod: "https://firefox.settings.services.mozilla.com",
-  stage: "https://firefox.settings.services.allizom.org",
-  dev: "https://remote-settings-dev.allizom.org",
-} as const;
-
-type RSEndpointKey = keyof typeof RS_ENDPOINTS;
+const QUERY_PARAM_RS_USE_PREVIEW = "rs_preview";
 
 /**
  * Get the RS environment from URL parameters, falling back to the defaults if
  * not specified.
  * @returns The RS environment key
  */
-function getRsEnv(): RSEndpointKey {
+function getRsEnv(): { env: RSEnvironment; usePreview: boolean } {
+  // Check if the environment is specified in the URL.
   const params = new URLSearchParams(window.location.search);
   const env = params.get(QUERY_PARAM_RS_ENV);
-  if (env && Object.keys(RS_ENDPOINTS).includes(env)) {
-    return env as RSEndpointKey;
+  const usePreview = params.get(QUERY_PARAM_RS_USE_PREVIEW) === "true";
+  if (isRSEnvValid(env)) {
+    return { env: env as RSEnvironment, usePreview };
   }
-  // Fall back to build env configuration or if env is not set, the default of "prod".
-  return (import.meta.env.VITE_RS_ENVIRONMENT as RSEndpointKey) || "prod";
+
+  // Fall back to build-time environment variable.
+  let viteEnv = import.meta.env.VITE_RS_ENVIRONMENT;
+  if (isRSEnvValid(viteEnv)) {
+    return { env: viteEnv as RSEnvironment, usePreview: false };
+  }
+
+  // Otherwise default to prod, non preview.
+  return { env: "prod", usePreview: false };
 }
 
 /**
  * Get the URL for the records endpoint for a given Remote Settings environment.
- * @param rsOrigin The origin of the Remote Settings environment.
+ * @param rsUrl The URL of the Remote Settings environment.
  * @returns The URL for the records endpoint.
  */
-function getRecordsUrl(rsOrigin: string): string {
+function getRecordsUrl(rsUrl: string): string {
   // Allow ENV to override the URL for testing.
   if (import.meta.env.VITE_RS_RECORDS_URL) {
     return import.meta.env.VITE_RS_RECORDS_URL;
   }
-  return `${rsOrigin}/v1/buckets/main/collections/url-classifier-exceptions/records`;
+  return rsUrl;
 }
 
 /**
  * Fetch the records from the Remote Settings environment.
- * @param rsOrigin The origin of the Remote Settings environment.
+ * @param rsUrl The URL of the Remote Settings environment.
  * @returns The records.
  */
-async function fetchRecords(rsOrigin: string): Promise<ExceptionListEntry[]> {
-  const response = await fetch(getRecordsUrl(rsOrigin));
+async function fetchRecords(rsUrl: string): Promise<ExceptionListEntry[]> {
+  const response = await fetch(getRecordsUrl(rsUrl));
   if (!response.ok) {
     throw new Error(`Failed to fetch records: ${response.statusText}`);
   }
@@ -124,7 +126,13 @@ export class App extends LitElement {
   // at build time. The user can change this via a dropdown. The user can also
   // override the environment via a query parameter.
   @state()
-  rsEnv: RSEndpointKey = getRsEnv();
+  rsEnv: RSEnvironment = "prod";
+
+  // Whether to use the preview environment. This bucket includes changes that
+  // are still pending review. The user can change this via a checkbox. It can
+  // also be overridden via a query parameter.
+  @state()
+  rsEnvUsePreview: boolean = false;
 
   // Holds all fetched records.
   @state()
@@ -187,6 +195,12 @@ export class App extends LitElement {
    */
   connectedCallback() {
     super.connectedCallback();
+
+    // Set the initial RS environment and preview setting.
+    let { env, usePreview } = getRsEnv();
+    this.rsEnv = env;
+    this.rsEnvUsePreview = usePreview;
+
     this.init();
   }
 
@@ -196,7 +210,9 @@ export class App extends LitElement {
   async init() {
     try {
       this.loading = true;
-      this.records = await fetchRecords(RS_ENDPOINTS[this.rsEnv]);
+
+      const urlStr = getRSEndpoint(this.rsEnv, this.rsEnvUsePreview).toString();
+      this.records = await fetchRecords(urlStr);
 
       // Spot check if the format is as expected.
       if (this.records.length && this.records[0].bugIds == null) {
@@ -227,6 +243,31 @@ export class App extends LitElement {
     return new Set(this.records.flatMap((record) => record.bugIds || [])).size;
   }
 
+  /**
+   * Handle changes to RS environment settings via the UI.
+   * @param event The change event either the dropdown or the checkbox.
+   */
+  private handleRSEnvChange(event: Event) {
+    const target = event.target as HTMLSelectElement | HTMLInputElement;
+
+    if (target.id === "rs-env") {
+      this.rsEnv = (target as HTMLSelectElement).value as RSEnvironment;
+      // Reset preview setting when environment changes.
+      this.rsEnvUsePreview = false;
+    } else if (target.id === "rs-env-preview") {
+      this.rsEnvUsePreview = (target as HTMLInputElement).checked;
+    }
+
+    // Update URL parameters to reflect current settings
+    const url = new URL(window.location.href);
+    url.searchParams.set(QUERY_PARAM_RS_ENV, this.rsEnv);
+    url.searchParams.set(QUERY_PARAM_RS_USE_PREVIEW, this.rsEnvUsePreview.toString());
+    window.history.pushState({}, "", url);
+
+    // Fetch the records again with the new settings
+    this.init();
+  }
+
   /**
    * Handle anchor navigation. We need to do this via JS because native navigation
    * does not traverse shadow DOM.
@@ -370,32 +411,28 @@ export class App extends LitElement {
         <footer>
           <p>
             <label for="rs-env">Remote Settings Environment:</label>
-            <select
-              id="rs-env"
-              @change=${(e: Event) => {
-                const newEnv = (e.target as HTMLSelectElement).value as RSEndpointKey;
-                this.rsEnv = newEnv;
-
-                // When the env changes reflect the update in the URL.
-                // Update URL parameter without reloading the page
-                const url = new URL(window.location.href);
-                url.searchParams.set(QUERY_PARAM_RS_ENV, newEnv);
-                window.history.pushState({}, "", url);
-
-                // Fetch the records again.
-                this.init();
-              }}
-            >
+            <select id="rs-env" @change=${this.handleRSEnvChange}>
               <option value="prod" ?selected=${this.rsEnv === "prod"}>Prod</option>
               <option value="stage" ?selected=${this.rsEnv === "stage"}>Stage</option>
               <option value="dev" ?selected=${this.rsEnv === "dev"}>Dev</option>
             </select>
+            <br />
+            <label for="rs-env-preview">Show changes pending review:</label>
+            <input
+              id="rs-env-preview"
+              type="checkbox"
+              ?checked=${this.rsEnvUsePreview}
+              @change=${this.handleRSEnvChange}
+            />
           </p>
           <p>
             Data source:
-            <a href="${getRecordsUrl(RS_ENDPOINTS[this.rsEnv])}"
-              >${getRecordsUrl(RS_ENDPOINTS[this.rsEnv])}</a
-            >.
+            ${(() => {
+              const recordsUrl = getRecordsUrl(
+                getRSEndpoint(this.rsEnv, this.rsEnvUsePreview).toString(),
+              );
+              return html`<a href="${recordsUrl}">${recordsUrl}</a>`;
+            })()}.
           </p>
         </footer>
         <!-- Show a link to the repository in the top right corner -->
