feat: Standardize browser profile display and proxy setting (#3004)

- Updates browser profile and workflow form controls to make
relationship between profile and proxy more apparent
- Displays suggested browser profiles based on workflow seed URLs
- Displays browser profile details when selecting a profile from a
workflow
- Fixes profile not saved to crawling defaults
- Consistency fixes for how proxies and crawler release channel is
displayed

---------

Co-authored-by: Ilya Kreymer <[email protected]>

Author: SuaYoo

.github/workflows/k3d-ci.yaml

@@ -45,6 +45,13 @@ jobs:
     needs: paths-filter
     if: needs.paths-filter.outputs.matches == 'true'
     steps:
+      - name: Initial Disk Cleanup
+        uses: mathio/gha-cleanup@v1
+        with:
+          remove-browsers: true
+          verbose: true
+
+
       - name: Create k3d Cluster
         uses: AbsaOSS/k3d-action@v2
         with:

frontend/src/components/ui/badge.ts

@@ -14,7 +14,8 @@ export type BadgeVariant =
   | "cyan"
   | "blue"
   | "high-contrast"
-  | "text";
+  | "text"
+  | "text-neutral";
 
 /**
  * Show numeric value in a label
@@ -66,6 +67,7 @@ export class Badge extends TailwindElement {
                   cyan: tw`bg-cyan-50 text-cyan-600 ring-cyan-600`,
                   blue: tw`bg-blue-50 text-blue-600 ring-blue-600`,
                   text: tw`text-blue-500 ring-blue-600`,
+                  "text-neutral": tw`text-neutral-500 ring-neutral-600`,
                 }[this.variant],
               ]
             : {
@@ -78,6 +80,7 @@ export class Badge extends TailwindElement {
                 cyan: tw`bg-cyan-50 text-cyan-600`,
                 blue: tw`bg-blue-50 text-blue-600`,
                 text: tw`text-blue-500`,
+                "text-neutral": tw`text-neutral-500`,
               }[this.variant],
           this.pill
             ? [

frontend/src/components/ui/config-details.ts

@@ -1,3 +1,4 @@
+import { consume } from "@lit/context";
 import { localized, msg, str } from "@lit/localize";
 import ISO6391 from "iso-639-1";
 import { html, nothing, type TemplateResult } from "lit";
@@ -6,6 +7,10 @@ import { when } from "lit/directives/when.js";
 import capitalize from "lodash/fp/capitalize";
 
 import { BtrixElement } from "@/classes/BtrixElement";
+import {
+  orgProxiesContext,
+  type OrgProxiesContext,
+} from "@/context/org-proxies";
 import { none, notSpecified } from "@/layouts/empty";
 import {
   Behavior,
@@ -36,6 +41,9 @@ import { getServerDefaults } from "@/utils/workflow";
 @customElement("btrix-config-details")
 @localized()
 export class ConfigDetails extends BtrixElement {
+  @consume({ context: orgProxiesContext, subscribe: true })
+  private readonly orgProxies?: OrgProxiesContext;
+
   @property({ type: Object })
   crawlConfig?: CrawlConfig;
 
@@ -235,23 +243,31 @@ export class ConfigDetails extends BtrixElement {
             msg("Browser Profile"),
             when(
               crawlConfig?.profileid,
-              () =>
-                html`<a
-                  class="text-blue-500 hover:text-blue-600"
+              () => html`
+                <btrix-link
                   href=${`${this.navigate.orgBasePath}/browser-profiles/profile/${
                     crawlConfig!.profileid
                   }`}
-                  @click=${this.navigate.link}
+                  hideIcon
                 >
                   ${crawlConfig?.profileName}
-                </a>`,
+                </btrix-link>
+              `,
               () =>
                 crawlConfig?.profileName ||
                 html`<span class="text-neutral-400"
                   >${msg("No custom profile")}</span
                 >`,
             ),
           )}
+          ${crawlConfig?.proxyId
+            ? this.renderSetting(
+                msg("Crawler Proxy Server"),
+                this.orgProxies?.servers.find(
+                  ({ id }) => id === crawlConfig.proxyId,
+                )?.label || capitalize(crawlConfig.proxyId),
+              )
+            : nothing}
           ${this.renderSetting(
             msg("Browser Windows"),
             crawlConfig?.browserWindows ? `${crawlConfig.browserWindows}` : "",
@@ -284,9 +300,6 @@ export class ConfigDetails extends BtrixElement {
                 ISO6391.getName(seedsConfig.lang),
               )
             : nothing}
-          ${crawlConfig?.proxyId
-            ? this.renderSetting(msg("Proxy"), capitalize(crawlConfig.proxyId))
-            : nothing}
         `,
       })}
       ${this.renderSection({

frontend/src/components/ui/link.ts

@@ -1,5 +1,5 @@
 import clsx from "clsx";
-import { html } from "lit";
+import { html, nothing } from "lit";
 import { customElement, property } from "lit/decorators.js";
 import { ifDefined } from "lit/directives/if-defined.js";
 
@@ -19,13 +19,16 @@ export class Link extends BtrixElement {
   @property({ type: String })
   variant: "primary" | "neutral" = "neutral";
 
+  @property({ type: Boolean })
+  hideIcon = false;
+
   render() {
     if (!this.href) return;
 
     return html`
       <a
         class=${clsx(
-          "group inline-flex items-center gap-1 transition-colors",
+          "group inline-flex items-center gap-1 transition-colors duration-fast",
           {
             primary: "text-primary-500 hover:text-primary-600",
             neutral: "text-blue-500 hover:text-blue-600",
@@ -39,12 +42,16 @@ export class Link extends BtrixElement {
           : this.navigate.link}
       >
         <slot></slot>
-        <sl-icon
-          slot="suffix"
-          name="arrow-right"
-          class="size-4 transition-transform group-hover:translate-x-1"
-        ></sl-icon
-      ></a>
+        ${this.hideIcon
+          ? nothing
+          : html`
+              <sl-icon
+                slot="suffix"
+                name="arrow-right"
+                class="size-4 transition-transform duration-fast group-hover:translate-x-1"
+              ></sl-icon>
+            `}
+      </a>
     `;
   }
 }

frontend/src/components/ui/select-crawler-proxy.ts

@@ -1,6 +1,6 @@
 import { localized, msg } from "@lit/localize";
 import type { SlSelect } from "@shoelace-style/shoelace";
-import { html } from "lit";
+import { html, type PropertyValues } from "lit";
 import { customElement, property, state } from "lit/decorators.js";
 import { ifDefined } from "lit/directives/if-defined.js";
 
@@ -40,15 +40,30 @@ export class SelectCrawlerProxy extends BtrixElement {
   @property({ type: String })
   defaultProxyId: string | null = null;
 
+  @property({ type: String })
+  profileProxyId?: string | null = null;
+
   @property({ type: Array })
   proxyServers: Proxy[] = [];
 
   @property({ type: String })
   proxyId: string | null = null;
 
+  @property({ type: String })
+  label?: string;
+
   @property({ type: String })
   size?: SlSelect["size"];
 
+  @property({ type: String })
+  placeholder?: string;
+
+  @property({ type: String })
+  helpText?: string;
+
+  @property({ type: Boolean })
+  disabled?: boolean;
+
   @state()
   private selectedProxy?: Proxy;
 
@@ -59,6 +74,18 @@ export class SelectCrawlerProxy extends BtrixElement {
     return this.selectedProxy?.id || "";
   }
 
+  protected willUpdate(changedProperties: PropertyValues): void {
+    if (changedProperties.has("proxyId")) {
+      if (this.proxyId) {
+        this.selectedProxy = this.proxyServers.find(
+          ({ id }) => id === this.proxyId,
+        );
+      } else if (changedProperties.get("proxyId")) {
+        this.selectedProxy = undefined;
+      }
+    }
+  }
+
   protected firstUpdated() {
     void this.initProxies();
   }
@@ -75,18 +102,21 @@ export class SelectCrawlerProxy extends BtrixElement {
     return html`
       <sl-select
         name="proxyId"
-        label=${msg("Crawler Proxy Server")}
+        label=${this.label || msg("Crawler Proxy Server")}
         value=${this.selectedProxy?.id || ""}
         placeholder=${this.defaultProxy
           ? `${msg(`Default Proxy:`)} ${this.defaultProxy.label}`
           : msg("No Proxy")}
         hoist
         clearable
+        ?disabled=${this.disabled ?? Boolean(this.profileProxyId)}
         size=${ifDefined(this.size)}
         @sl-change=${this.onChange}
         @sl-hide=${this.stopProp}
         @sl-after-hide=${this.stopProp}
       >
+        <slot name="prefix" slot="prefix"></slot>
+        <slot name="suffix" slot="suffix"></slot>
         ${this.proxyServers.map(
           (server) =>
             html` <sl-option value=${server.id}>
@@ -118,6 +148,7 @@ export class SelectCrawlerProxy extends BtrixElement {
               </div>
             `
           : ``}
+        <slot name="help-text" slot="help-text"></slot>
       </sl-select>
     `;
   }

frontend/src/controllers/localize.ts

@@ -55,7 +55,11 @@ export class LocalizeController extends SlLocalizeController {
               })
             : seconds > 60
               ? html`<sl-relative-time
-                  class=${ifDefined(capitalize ? tw`capitalize` : undefined)}
+                  class=${ifDefined(
+                    capitalize
+                      ? tw`inline-block first-letter:capitalize`
+                      : undefined,
+                  )}
                   sync
                   date=${dateStr}
                 ></sl-relative-time>`

frontend/src/features/browser-profiles/new-browser-profile-dialog.ts

@@ -116,50 +116,53 @@ export class NewBrowserProfileDialog extends BtrixElement {
           >
           </btrix-url-input>
 
+          ${showProxies
+            ? html`
+                <div class="mt-4">
+                  <btrix-select-crawler-proxy
+                    .label=${msg("Proxy Server")}
+                    defaultProxyId=${ifDefined(
+                      this.defaultProxyId || undefined,
+                    )}
+                    .proxyServers=${proxyServers}
+                    .proxyId="${this.proxyId || ""}"
+                    @btrix-change=${(e: SelectCrawlerProxyChangeEvent) =>
+                      (this.proxyId = e.detail.value)}
+                  >
+                    <div slot="help-text">
+                      ${msg(
+                        "When a proxy is selected, websites will see traffic as coming from the IP address of the proxy rather than where Browsertrix is deployed.",
+                      )}
+                    </div>
+                  </btrix-select-crawler-proxy>
+                </div>
+              `
+            : nothing}
+
           <sl-input
             class="mt-4"
             label=${msg("Profile Name")}
             name="profile-name"
             placeholder=${msg("example.com")}
             value=${ifDefined(this.defaultUrl)}
-            help-text=${msg("Defaults to site's domain name if omitted.")}
+            help-text=${msg(
+              "Defaults to the primary site's domain name if omitted.",
+            )}
             maxlength="50"
           >
           </sl-input>
 
-          ${when(
-            showChannels || showProxies,
-            () => html`
-              <btrix-details class="mt-4" open>
-                <span slot="title">${msg("Crawler Settings")}</span>
-
-                ${showChannels
-                  ? html`<div class="mt-4">
-                      <btrix-select-crawler
-                        .crawlerChannel=${this.crawlerChannel}
-                        @on-change=${(e: SelectCrawlerChangeEvent) =>
-                          (this.crawlerChannel = e.detail.value!)}
-                      ></btrix-select-crawler>
-                    </div>`
-                  : nothing}
-                ${showProxies
-                  ? html`
-                      <div class="mt-4">
-                        <btrix-select-crawler-proxy
-                          defaultProxyId=${ifDefined(
-                            this.defaultProxyId || undefined,
-                          )}
-                          .proxyServers=${proxyServers}
-                          .proxyId="${this.proxyId || ""}"
-                          @btrix-change=${(e: SelectCrawlerProxyChangeEvent) =>
-                            (this.proxyId = e.detail.value)}
-                        ></btrix-select-crawler-proxy>
-                      </div>
-                    `
-                  : nothing}
-              </btrix-details>
-            `,
-          )}
+          ${showChannels
+            ? html`<btrix-details class="mt-4">
+                <span slot="title">${msg("Browser Session Settings")}</span>
+                <div class="mt-4">
+                  <btrix-select-crawler
+                    .crawlerChannel=${this.crawlerChannel}
+                    @on-change=${(e: SelectCrawlerChangeEvent) =>
+                      (this.crawlerChannel = e.detail.value!)}
+                  ></btrix-select-crawler></div
+              ></btrix-details>`
+            : nothing}
 
           <input class="invisible block size-0" type="submit" />
         </form>