diff --git a/backend/btrixcloud/crawls.py b/backend/btrixcloud/crawls.py index c6c8780f06..bb986b45bf 100644 --- a/backend/btrixcloud/crawls.py +++ b/backend/btrixcloud/crawls.py @@ -291,6 +291,8 @@ async def list_crawls( "qaRunCount", "lastQAState", "lastQAStarted", + "crawlExecSeconds", + "pageCount", ): raise HTTPException(status_code=400, detail="invalid_sort_by") if sort_direction not in (1, -1): diff --git a/frontend/docs/docs/user-guide/archived-items.md b/frontend/docs/docs/user-guide/archived-items.md index eefe4288ff..a226f03658 100644 --- a/frontend/docs/docs/user-guide/archived-items.md +++ b/frontend/docs/docs/user-guide/archived-items.md @@ -15,10 +15,10 @@ The status of an archived item depends on its type. Uploads will always have the | :bootstrap-check-circle-fill: Complete | The crawl completed according to the workflow's settings. Workflows with [crawl limits](workflow-setup.md#crawl-limits) set may stop running before they capture every queued page, but the resulting archived item will still be marked as "Complete". | | :bootstrap-dash-square-fill: Stopped | The crawl workflow was _stopped_ gracefully by a user and data is saved. | | :bootstrap-exclamation-square-fill: Stopped: Reason | A workflow limit (listed as the reason) was reached and data is saved. | -| :bootstrap-x-octagon-fill: Canceled | The crawl workflow was _canceled_ by a user, no data is saved. | +| :bootstrap-exclamation-octagon: Canceled | The crawl workflow was _canceled_ by a user, no data is saved. | | :bootstrap-diamond-triangle-fill: Failed | A serious error occurred while crawling, no data is saved.| -Because :bootstrap-x-octagon-fill: Canceled and :bootstrap-exclamation-diamond-fill: Failed crawls do not contain data, they are omitted from the archived items list page and cannot be added to a collection. +Because :bootstrap-exclamation-octagon: Canceled and :bootstrap-exclamation-diamond-fill: Failed crawls do not contain data, they are omitted from the archived items list page and cannot be added to a collection. ## Archived Item Details diff --git a/frontend/src/components/ui/badge.ts b/frontend/src/components/ui/badge.ts index f5d7144a5c..16bb776352 100644 --- a/frontend/src/components/ui/badge.ts +++ b/frontend/src/components/ui/badge.ts @@ -28,6 +28,9 @@ export class Badge extends TailwindElement { @property({ type: String }) variant: BadgeVariant = "neutral"; + @property({ type: String }) + size?: "medium" | "large" = "medium"; + @property({ type: Boolean }) outline = false; @@ -47,15 +50,16 @@ export class Badge extends TailwindElement { return html` diff --git a/frontend/src/events/btrix-cancel.ts b/frontend/src/events/btrix-cancel.ts new file mode 100644 index 0000000000..52eb29f6f1 --- /dev/null +++ b/frontend/src/events/btrix-cancel.ts @@ -0,0 +1,7 @@ +export type BtrixCancelEvent = CustomEvent; + +declare global { + interface GlobalEventHandlersEventMap { + "btrix-cancel": BtrixCancelEvent; + } +} diff --git a/frontend/src/events/btrix-confirm.ts b/frontend/src/events/btrix-confirm.ts new file mode 100644 index 0000000000..e6a6ec2e1f --- /dev/null +++ b/frontend/src/events/btrix-confirm.ts @@ -0,0 +1,7 @@ +export type BtrixConfirmEvent = CustomEvent; + +declare global { + interface GlobalEventHandlersEventMap { + "btrix-confirm": BtrixConfirmEvent; + } +} diff --git a/frontend/src/features/archived-items/archived-item-state-filter.ts b/frontend/src/features/archived-items/archived-item-state-filter.ts index 541db1e4b7..d64e6990ef 100644 --- a/frontend/src/features/archived-items/archived-item-state-filter.ts +++ b/frontend/src/features/archived-items/archived-item-state-filter.ts @@ -21,7 +21,7 @@ import { CrawlStatus } from "./crawl-status"; import { BtrixElement } from "@/classes/BtrixElement"; import type { BtrixChangeEvent } from "@/events/btrix-change"; -import { type CrawlState } from "@/types/crawlState"; +import { CRAWL_STATES, type CrawlState } from "@/types/crawlState"; import { finishedCrawlStates } from "@/utils/crawler"; import { isNotEqual } from "@/utils/is-not-equal"; import { tw } from "@/utils/tailwind"; @@ -42,6 +42,9 @@ export class ArchivedItemStateFilter extends BtrixElement { @property({ type: Array }) states?: CrawlState[]; + @property({ type: Boolean }) + showUnfinishedStates = false; + @state() private searchString = ""; @@ -51,11 +54,19 @@ export class ArchivedItemStateFilter extends BtrixElement { @queryAll("sl-checkbox") private readonly checkboxes!: NodeListOf; - private readonly fuse = new Fuse(finishedCrawlStates); + private fuse = new Fuse(finishedCrawlStates); @state({ hasChanged: isNotEqual }) selected = new Map(); + connectedCallback(): void { + super.connectedCallback(); + + if (this.showUnfinishedStates) { + this.fuse = new Fuse(CRAWL_STATES); + } + } + protected willUpdate(changedProperties: PropertyValues): void { if (changedProperties.has("states")) { if (this.states) { @@ -85,7 +96,9 @@ export class ArchivedItemStateFilter extends BtrixElement { render() { const options = this.searchString ? this.fuse.search(this.searchString) - : finishedCrawlStates.map((state) => ({ item: state })); + : (this.showUnfinishedStates ? CRAWL_STATES : finishedCrawlStates).map( + (state) => ({ item: state }), + ); return html` `; diff --git a/frontend/src/features/archived-items/delete-item-dialog.ts b/frontend/src/features/archived-items/delete-item-dialog.ts new file mode 100644 index 0000000000..8d84a455f0 --- /dev/null +++ b/frontend/src/features/archived-items/delete-item-dialog.ts @@ -0,0 +1,144 @@ +import { localized, msg, str } from "@lit/localize"; +import { Task } from "@lit/task"; +import { html } from "lit"; +import { customElement, property, query } from "lit/decorators.js"; +import { when } from "lit/directives/when.js"; + +import { BtrixElement } from "@/classes/BtrixElement"; +import type { Dialog } from "@/components/ui/dialog"; +import type { ArchivedItem } from "@/types/crawler"; +import { isCrawl, renderName } from "@/utils/crawler"; +import { pluralOf } from "@/utils/pluralize"; + +/** + * Confirm deletion of an archived item, crawl associated + * with an archived item, or crawl run. + * + * @slot name + */ +@customElement("btrix-delete-item-dialog") +@localized() +export class DeleteItemDialog extends BtrixElement { + @property({ type: Object }) + item?: ArchivedItem; + + @property({ type: Boolean }) + open = false; + + @query("btrix-dialog") + readonly dialog?: Dialog | null; + + private readonly collectionsTask = new Task(this, { + task: async ([open, crawl], { signal }) => { + if (!crawl?.collectionIds) return; + + if (!open) { + return crawl.collectionIds.map((id) => ({ id })); + } + + return (await this.getCrawl(crawl.id, signal)).collections; + }, + args: () => [this.open, this.item] as const, + }); + + render() { + return html` + ${this.renderContent()} + `; + } + + private renderContent() { + const item = this.item; + + if (!item) return; + + const crawl = isCrawl(item); + const item_name = html`${renderName(item)}`; + + return html` +

+ ${msg(html`Are you sure you want to delete ${item_name}?`)} + ${msg("All associated files and logs will be deleted.")} +

+ + ${this.renderCollections()} +
+ { + void this.dialog?.hide(); + this.dispatchEvent(new CustomEvent("btrix-cancel")); + }} + >${msg("Cancel")} + { + this.dispatchEvent(new CustomEvent("btrix-confirm")); + }} + >${crawl + ? msg("Delete Crawl") + : msg("Delete Archived Item")} +
+ `; + } + + private renderCollections() { + if (!this.item?.collectionIds.length) return; + + const { collectionIds } = this.item; + const count = collectionIds.length; + + const number_of_collections = this.localize.number(count); + const plural_of_collections = pluralOf("collections", count); + + return html` +

+ ${msg( + str`The archived item will be removed from ${number_of_collections} ${plural_of_collections}:`, + )} +

+ ${this.collectionsTask.render({ + pending: () => + html` ({ id }))} + baseUrl="${this.navigate.orgBasePath}/collections/view" + > + `, + complete: (res) => + when( + res, + (collections) => + html` + `, + ), + })} + `; + } + + private async getCrawl(id: string, signal: AbortSignal) { + const data: ArchivedItem = await this.api.fetch( + `/orgs/${this.orgId}/crawls/${id}/replay.json`, + { + signal, + }, + ); + + return data; + } +} diff --git a/frontend/src/features/archived-items/index.ts b/frontend/src/features/archived-items/index.ts index 4540bac898..702ce361e9 100644 --- a/frontend/src/features/archived-items/index.ts +++ b/frontend/src/features/archived-items/index.ts @@ -1,9 +1,9 @@ import("./archived-item-list"); import("./archived-item-state-filter"); import("./archived-item-tag-filter"); -import("./crawl-list"); import("./crawl-log-table"); import("./crawl-logs"); +import("./delete-item-dialog"); import("./item-metadata-editor"); import("./crawl-pending-exclusions"); import("./crawl-queue"); diff --git a/frontend/src/features/crawl-workflows/index.ts b/frontend/src/features/crawl-workflows/index.ts index 2dd12c1a8a..8a662816b2 100644 --- a/frontend/src/features/crawl-workflows/index.ts +++ b/frontend/src/features/crawl-workflows/index.ts @@ -9,6 +9,7 @@ import("./workflow-action-menu/workflow-action-menu"); import("./workflow-editor"); import("./workflow-list"); import("./workflow-schedule-filter"); +import("./workflow-search"); import("./workflow-tag-filter"); import("./workflow-profile-filter"); import("./workflow-last-crawl-state-filter"); diff --git a/frontend/src/features/crawl-workflows/workflow-list.ts b/frontend/src/features/crawl-workflows/workflow-list.ts index b800d03111..00a9327e81 100644 --- a/frontend/src/features/crawl-workflows/workflow-list.ts +++ b/frontend/src/features/crawl-workflows/workflow-list.ts @@ -452,7 +452,12 @@ export class WorkflowListItem extends BtrixElement {
- ${workflow.modifiedByName} + ${workflow.modifiedByName + ? html`` + : notSpecified}
${shortDate(workflow.modified)}
diff --git a/frontend/src/features/crawl-workflows/workflow-search.ts b/frontend/src/features/crawl-workflows/workflow-search.ts new file mode 100644 index 0000000000..c6cfb74314 --- /dev/null +++ b/frontend/src/features/crawl-workflows/workflow-search.ts @@ -0,0 +1,26 @@ +import { localized, msg } from "@lit/localize"; +import { customElement, property } from "lit/decorators.js"; + +import { SearchCombobox } from "@/components/ui/search-combobox"; + +export type SearchFields = "name" | "firstSeed"; + +@customElement("btrix-workflow-search") +@localized() +export class WorkflowSearch extends SearchCombobox<{ [x: string]: string }> { + static FieldLabels: Record = { + name: msg("Name"), + firstSeed: msg("Crawl Start URL"), + }; + + @property({ type: Array }) + searchOptions: { [x: string]: string }[] = []; + + @property({ type: String }) + selectedKey?: string; + + readonly searchKeys = ["name", "firstSeed"]; + readonly keyLabels = WorkflowSearch.FieldLabels; + + placeholder = msg("Search by workflow name or crawl start URL"); +} diff --git a/frontend/src/features/archived-items/crawl-list.ts b/frontend/src/features/crawls/crawl-list/crawl-list-item.ts similarity index 51% rename from frontend/src/features/archived-items/crawl-list.ts rename to frontend/src/features/crawls/crawl-list/crawl-list-item.ts index b162fa510f..3edd7fe5c4 100644 --- a/frontend/src/features/archived-items/crawl-list.ts +++ b/frontend/src/features/crawls/crawl-list/crawl-list-item.ts @@ -1,31 +1,12 @@ -/** - * Display list of crawls - * - * Usage example: - * ```ts - * - * - * - * - * - * - * ``` - */ import { localized, msg } from "@lit/localize"; import { css, html, nothing, type TemplateResult } from "lit"; -import { - customElement, - property, - query, - queryAssignedElements, -} from "lit/decorators.js"; +import { customElement, property, query } from "lit/decorators.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { BtrixElement } from "@/classes/BtrixElement"; -import { TailwindElement } from "@/classes/TailwindElement"; import type { OverflowDropdown } from "@/components/ui/overflow-dropdown"; import type { Crawl } from "@/types/crawler"; -import { renderName } from "@/utils/crawler"; +import { isSkipped, renderName } from "@/utils/crawler"; import { humanizeExecutionSeconds } from "@/utils/executionTimeFormatter"; /** @@ -99,7 +80,7 @@ export class CrawlListItem extends BtrixElement { `; } else { const label = html` - + ${this.safeRender((workflow) => renderName(workflow))} `; @@ -114,6 +95,12 @@ export class CrawlListItem extends BtrixElement { `; } + const skipped = isSkipped(this.crawl); + const hasExec = Boolean(this.crawl.crawlExecSeconds); + const notApplicable = html` + + `; + return html` ` - : html`---`, - )} - - - ${this.safeRender((crawl) => - this.localize.humanizeDuration( - (crawl.finished - ? new Date(crawl.finished) - : new Date() - ).valueOf() - new Date(crawl.started).valueOf(), - ), + : notApplicable, )} ${this.safeRender((crawl) => - humanizeExecutionSeconds(crawl.crawlExecSeconds), + !skipped + ? html` + ${humanizeExecutionSeconds(crawl.crawlExecSeconds, { + style: "short", + })} + + ${humanizeExecutionSeconds(crawl.crawlExecSeconds, { + style: "long", + })} + + ` + : notApplicable, )} ${this.safeRender((crawl) => { - const pagesFound = +(crawl.stats?.found || 0); - if (crawl.finished) { - const pagesComplete = crawl.pageCount ? +crawl.pageCount : 0; - return `${this.localize.number(pagesComplete, { notation: "compact" })}`; + if (hasExec) { + const pagesComplete = crawl.finished + ? crawl.pageCount + ? +crawl.pageCount + : 0 + : +(crawl.stats?.done || 0); + + return this.localize.number(pagesComplete, { + notation: "compact", + }); } - const pagesComplete = +(crawl.stats?.done || 0); - return `${this.localize.number(pagesComplete, { notation: "compact" })} / ${this.localize.number(pagesFound, { notation: "compact" })}`; + return notApplicable; })} ${this.safeRender((crawl) => - this.localize.bytes( - crawl.finished ? crawl.fileSize || 0 : +(crawl.stats?.size || 0), - { - unitDisplay: "narrow", - }, - ), + hasExec + ? this.localize.bytes( + crawl.finished + ? crawl.fileSize || 0 + : +(crawl.stats?.size || 0), + ) + : notApplicable, )} -
- ${this.safeRender((crawl) => crawl.userName)} -
+ ${this.safeRender( + (crawl) => + html``, + )} ${this.renderActions()} @@ -248,110 +247,3 @@ export class CrawlListItem extends BtrixElement { `; } } - -/** - * @slot - */ -@customElement("btrix-crawl-list") -@localized() -export class CrawlList extends TailwindElement { - static styles = css` - btrix-table { - --btrix-table-cell-gap: var(--sl-spacing-x-small); - --btrix-table-cell-padding-x: var(--sl-spacing-small); - } - - btrix-table-body { - --btrix-table-cell-padding-y: var(--sl-spacing-2x-small); - } - - btrix-table-body ::slotted(*:nth-of-type(n + 2)) { - --btrix-border-top: 1px solid var(--sl-panel-border-color); - } - - btrix-table-body ::slotted(*:first-of-type) { - --btrix-border-radius-top: var(--sl-border-radius-medium); - } - - btrix-table-body ::slotted(*:last-of-type) { - --btrix-border-radius-bottom: var(--sl-border-radius-medium); - } - `; - - @property({ type: String }) - collectionId?: string; - - @property({ type: String }) - workflowId?: string; - - @queryAssignedElements({ selector: "btrix-crawl-list-item" }) - listItems!: HTMLElement[]; - - render() { - return html` - - - - - ${msg("Status")} - - ${this.workflowId - ? nothing - : html` - - ${msg("Name")} - - `} - - ${msg("Started")} - - - ${msg("Finished")} - - ${msg("Run Duration")} - ${msg("Execution Time")} - ${msg("Pages")} - ${msg("Size")} - - ${msg("Run By")} - - - ${msg("Row actions")} - - - - - - - `; - } - - private handleSlotchange() { - const assignProp = ( - el: HTMLElement, - attr: { name: string; value: string }, - ) => { - if (!el.attributes.getNamedItem(attr.name)) { - el.setAttribute(attr.name, attr.value); - } - }; - - this.listItems.forEach((item) => { - assignProp(item, { - name: "collectionId", - value: this.collectionId || "", - }); - assignProp(item, { name: "workflowId", value: this.workflowId || "" }); - }); - } -} diff --git a/frontend/src/features/crawls/crawl-list/crawl-list.ts b/frontend/src/features/crawls/crawl-list/crawl-list.ts new file mode 100644 index 0000000000..2b515fc4b5 --- /dev/null +++ b/frontend/src/features/crawls/crawl-list/crawl-list.ts @@ -0,0 +1,128 @@ +import { localized, msg } from "@lit/localize"; +import { css, html, nothing } from "lit"; +import { + customElement, + property, + queryAssignedElements, +} from "lit/decorators.js"; + +import { TailwindElement } from "@/classes/TailwindElement"; + +/** + * Display list of crawls + * + * Usage example: + * ```ts + * + * + * + * + * + * + * ``` + * + * @slot + */ +@customElement("btrix-crawl-list") +@localized() +export class CrawlList extends TailwindElement { + static styles = css` + btrix-table { + --btrix-table-cell-gap: var(--sl-spacing-x-small); + --btrix-table-cell-padding-x: var(--sl-spacing-small); + } + + btrix-table-body { + --btrix-table-cell-padding-y: var(--sl-spacing-2x-small); + } + + btrix-table-body ::slotted(*:nth-of-type(n + 2)) { + --btrix-border-top: 1px solid var(--sl-panel-border-color); + } + + btrix-table-body ::slotted(*:first-of-type) { + --btrix-border-radius-top: var(--sl-border-radius-medium); + } + + btrix-table-body ::slotted(*:last-of-type) { + --btrix-border-radius-bottom: var(--sl-border-radius-medium); + } + `; + + @property({ type: String }) + collectionId?: string; + + @property({ type: String }) + workflowId?: string; + + @queryAssignedElements({ selector: "btrix-crawl-list-item" }) + listItems!: HTMLElement[]; + + render() { + const columns = [ + "min-content [clickable-start]", + this.workflowId ? undefined : "36ch", // Name + "22ch", // Started + "22ch", // Finished + "auto", // Execution time + "auto", // Pages + "auto", // Size + "[clickable-end] 20ch", // Run by + "min-content", + ] + .filter((v) => v) + .join(" "); + + return html` + + + + ${msg("Status")} + + ${this.workflowId + ? nothing + : html` + + ${msg("Name")} + + `} + ${msg("Started")} + + ${msg("Finished")} + + ${msg("Execution Time")} + ${msg("Pages")} + ${msg("Size")} + ${msg("Run By")} + + ${msg("Row actions")} + + + + + + + `; + } + + private handleSlotchange() { + const assignProp = ( + el: HTMLElement, + attr: { name: string; value: string }, + ) => { + if (!el.attributes.getNamedItem(attr.name)) { + el.setAttribute(attr.name, attr.value); + } + }; + + this.listItems.forEach((item) => { + assignProp(item, { + name: "collectionId", + value: this.collectionId || "", + }); + assignProp(item, { name: "workflowId", value: this.workflowId || "" }); + }); + } +} diff --git a/frontend/src/features/crawls/crawl-list/index.ts b/frontend/src/features/crawls/crawl-list/index.ts new file mode 100644 index 0000000000..39e87f9901 --- /dev/null +++ b/frontend/src/features/crawls/crawl-list/index.ts @@ -0,0 +1,2 @@ +import "./crawl-list"; +import "./crawl-list-item"; diff --git a/frontend/src/features/crawls/index.ts b/frontend/src/features/crawls/index.ts new file mode 100644 index 0000000000..1e525d2ee6 --- /dev/null +++ b/frontend/src/features/crawls/index.ts @@ -0,0 +1 @@ +import("./crawl-list"); diff --git a/frontend/src/features/index.ts b/frontend/src/features/index.ts index 635b4a9669..bd047b5664 100644 --- a/frontend/src/features/index.ts +++ b/frontend/src/features/index.ts @@ -1,9 +1,11 @@ import "./accounts"; import "./archived-items"; import "./browser-profiles"; +import "./crawls"; import "./collections"; import "./crawl-workflows"; import "./org"; import "./qa"; +import "./users"; import("./admin"); diff --git a/frontend/src/features/users/index.ts b/frontend/src/features/users/index.ts new file mode 100644 index 0000000000..f1e6ac3b44 --- /dev/null +++ b/frontend/src/features/users/index.ts @@ -0,0 +1 @@ +import("./user-chip"); diff --git a/frontend/src/features/users/user-chip.ts b/frontend/src/features/users/user-chip.ts new file mode 100644 index 0000000000..9fb94d2cdb --- /dev/null +++ b/frontend/src/features/users/user-chip.ts @@ -0,0 +1,29 @@ +import { localized } from "@lit/localize"; +import { html } from "lit"; +import { customElement, property } from "lit/decorators.js"; + +import { BtrixElement } from "@/classes/BtrixElement"; + +@customElement("btrix-user-chip") +@localized() +export class UserChip extends BtrixElement { + @property({ type: String }) + userId?: string; + + @property({ type: String }) + userName?: string; + + render() { + if (!this.userName) return; + + return html` + ${this.userName} + `; + } +} diff --git a/frontend/src/index.ts b/frontend/src/index.ts index f159b684da..0a3674c32d 100644 --- a/frontend/src/index.ts +++ b/frontend/src/index.ts @@ -863,7 +863,8 @@ export class App extends BtrixElement { const orgTab = pathname .slice(pathname.indexOf(slug) + slug.length) .replace(/(^\/|\/$)/, "") - .split("/")[0]; + .split("/")[0] + .split("?")[0]; return html` (this.openDialogName = undefined)} @updated=${() => void this.fetchCrawl()} > + + (this.openDialogName = undefined)} + @btrix-confirm=${() => { + this.openDialogName = undefined; + void this.deleteCrawl(); + }} + > + ${this.item?.finished && isCrawl(this.item) + ? html`${renderName(this.item)} + (${this.localize.date(this.item.finished)})` + : nothing} + `; } @@ -508,7 +525,7 @@ export class ArchivedItemDetail extends BtrixElement { breadcrumbs.push( { href: `${this.navigate.orgBasePath}/workflows`, - content: msg("Crawl Workflows"), + content: msg("Workflows"), }, { href: `${this.navigate.orgBasePath}/workflows/${this.item?.cid}`, @@ -768,8 +785,14 @@ export class ArchivedItemDetail extends BtrixElement { () => html` void this.deleteCrawl()} + class="menu-item-danger" + @click=${() => { + if (isSuccess) { + this.openDialogName = "delete"; + } else { + void this.deleteCrawl(); + } + }} > ${isWorkflowCrawl @@ -1362,14 +1385,7 @@ export class ArchivedItemDetail extends BtrixElement { return !formEl.querySelector("[data-invalid]"); } - // TODO replace with in-page dialog private async deleteCrawl() { - if ( - !window.confirm(msg(str`Are you sure you want to delete this crawl?`)) - ) { - return; - } - try { const _data = await this.api.fetch( `/orgs/${this.item!.oid}/${ diff --git a/frontend/src/pages/org/archived-items.ts b/frontend/src/pages/org/archived-items.ts index c273895958..7ce6f8586a 100644 --- a/frontend/src/pages/org/archived-items.ts +++ b/frontend/src/pages/org/archived-items.ts @@ -1,6 +1,6 @@ import { localized, msg, str } from "@lit/localize"; import { Task } from "@lit/task"; -import type { SlButton, SlSelect } from "@shoelace-style/shoelace"; +import type { SlSelect } from "@shoelace-style/shoelace"; import { html, nothing, type PropertyValues } from "lit"; import { customElement, property, query, state } from "lit/decorators.js"; import { ifDefined } from "lit/directives/if-defined.js"; @@ -11,7 +11,6 @@ import queryString from "query-string"; import type { ArchivedItem, Crawl, Workflow } from "./types"; import { BtrixElement } from "@/classes/BtrixElement"; -import type { Dialog } from "@/components/ui/dialog"; import { type BtrixFilterChipChangeEvent, type FilterChip, @@ -33,7 +32,9 @@ import { isApiError } from "@/utils/api"; import { finishedCrawlStates, isActive, + isCrawl, isSuccessfullyFinished, + renderName, } from "@/utils/crawler"; import { isArchivingDisabled } from "@/utils/orgs"; import { tw } from "@/utils/tailwind"; @@ -101,10 +102,9 @@ type FilterBy = { }; /** - * Usage: - * ```ts - * - * ``` + * An archived item can be any replayable crawl (i.e. crawl with WACZ files) + * or an uploaded WACZ. Users can view, search, and filter archived items + * from this page. */ @customElement("btrix-archived-items") @localized() @@ -415,17 +415,18 @@ export class CrawlsList extends BtrixElement { }[] = [ { itemType: null, - label: msg("All"), + icon: "file-zip-fill", + label: msg("All Items"), }, { itemType: "crawl", icon: "gear-wide-connected", - label: msg("Crawls"), + label: msg("Crawled Items"), }, { itemType: "upload", icon: "upload", - label: msg("Uploads"), + label: msg("Uploaded Items"), }, ]; @@ -458,8 +459,7 @@ export class CrawlsList extends BtrixElement { ${listTypes.map(({ label, itemType, icon }) => { const isSelected = itemType === this.itemType; return html` (this.isDeletingItem = false)} + @btrix-confirm=${async () => { + this.isDeletingItem = false; + if (this.itemToDelete) { + await this.deleteItem(this.itemToDelete); + } + }} > - ${msg("This item will be removed from any Collection it is a part of.")} - ${when(this.itemToDelete?.type === "crawl", () => - msg( - "All files and logs associated with this item will also be deleted, and the crawl will no longer be visible in its associated Workflow.", - ), - )} -
- - void (e.currentTarget as SlButton) - .closest("btrix-dialog") - ?.hide()} - >${msg("Cancel")} - { - this.isDeletingItem = false; - if (this.itemToDelete) { - await this.deleteItem(this.itemToDelete); - } - }} - >${msg( - str`Delete ${ - this.itemToDelete?.type === "upload" - ? msg("Upload") - : msg("Crawl") - }`, - )} -
- + ${this.itemToDelete?.finished && isCrawl(this.itemToDelete) + ? html`${renderName(this.itemToDelete)} + (${this.localize.date(this.itemToDelete.finished)})` + : nothing} + `; private renderControls() { @@ -858,7 +836,7 @@ export class CrawlsList extends BtrixElement { () => html` this.confirmDeleteItem(item)} > diff --git a/frontend/src/pages/org/crawling.ts b/frontend/src/pages/org/crawling.ts new file mode 100644 index 0000000000..0b1af26036 --- /dev/null +++ b/frontend/src/pages/org/crawling.ts @@ -0,0 +1,151 @@ +import { localized, msg } from "@lit/localize"; +import type { SlSelectEvent } from "@shoelace-style/shoelace"; +import { html } from "lit"; +import { customElement, property } from "lit/decorators.js"; +import { when } from "lit/directives/when.js"; + +import { ScopeType } from "./types"; + +import { BtrixElement } from "@/classes/BtrixElement"; +import type { SelectJobTypeEvent } from "@/features/crawl-workflows/new-workflow-dialog"; +import { pageHeader } from "@/layouts/pageHeader"; +import { OrgTab, WorkflowTab } from "@/routes"; +import scopeTypeLabels from "@/strings/crawl-workflows/scopeType"; +import { NewWorkflowOnlyScopeType } from "@/types/workflow"; +import { tw } from "@/utils/tailwind"; + +@customElement("btrix-org-crawling") +@localized() +export class OrgCrawling extends BtrixElement { + @property({ type: String }) + crawlingTab?: string; + + render() { + return html` + ${pageHeader({ + title: msg("Crawling"), + actions: html` + ${when( + this.appState.isAdmin, + () => + html` + + `, + )} + ${when( + this.appState.isCrawler, + () => html` + + + this.navigate.to( + `${this.navigate.orgBasePath}/workflows/new`, + { + scopeType: + this.appState.userPreferences?.newWorkflowScopeType, + }, + )} + > + + ${msg("New Workflow")} + { + const { value } = e.detail.item; + + if (value) { + this.dispatchEvent( + new CustomEvent( + "select-job-type", + { + detail: value as SelectJobTypeEvent["detail"], + }, + ), + ); + } + }} + > + + ${msg("Scope options")} + + + ${msg("Page Crawl")} + ${scopeTypeLabels[ScopeType.Page]} + + ${scopeTypeLabels[NewWorkflowOnlyScopeType.PageList]} + + + ${scopeTypeLabels[ScopeType.SPA]} + + + ${msg("Site Crawl")} + + ${scopeTypeLabels[ScopeType.Prefix]} + + + ${scopeTypeLabels[ScopeType.Host]} + + + ${scopeTypeLabels[ScopeType.Domain]} + + + ${scopeTypeLabels[ScopeType.Custom]} + + + + + `, + )} + `, + classNames: tw`mb-3`, + })} +
+ + + ${msg("Workflows")} + + + + ${msg("Crawl Runs")} + +
+ + ${when( + this.crawlingTab === WorkflowTab.Crawls, + () => html``, + () => html``, + )} + `; + } +} diff --git a/frontend/src/pages/org/crawls.ts b/frontend/src/pages/org/crawls.ts new file mode 100644 index 0000000000..803010fc7f --- /dev/null +++ b/frontend/src/pages/org/crawls.ts @@ -0,0 +1,873 @@ +import { localized, msg, str } from "@lit/localize"; +import { Task } from "@lit/task"; +import type { SlSelect } from "@shoelace-style/shoelace"; +import { html, nothing, type PropertyValues } from "lit"; +import { customElement, query, state } from "lit/decorators.js"; +import { ifDefined } from "lit/directives/if-defined.js"; +import { repeat } from "lit/directives/repeat.js"; +import { when } from "lit/directives/when.js"; +import queryString from "query-string"; + +import type { Crawl, Workflow } from "./types"; + +import { BtrixElement } from "@/classes/BtrixElement"; +import { + type BtrixFilterChipChangeEvent, + type FilterChip, +} from "@/components/ui/filter-chip"; +import { + parsePage, + type PageChangeEvent, + type Pagination, +} from "@/components/ui/pagination"; +import type { SelectEvent } from "@/components/ui/search-combobox"; +import { ClipboardController } from "@/controllers/clipboard"; +import { SearchParamsValue } from "@/controllers/searchParamsValue"; +import { type BtrixChangeArchivedItemStateFilterEvent } from "@/features/archived-items/archived-item-state-filter"; +import { + WorkflowSearch, + type SearchFields, +} from "@/features/crawl-workflows/workflow-search"; +import { OrgTab } from "@/routes"; +import type { APIPaginatedList, APIPaginationQuery } from "@/types/api"; +import type { CrawlState } from "@/types/crawlState"; +import { isApiError } from "@/utils/api"; +import { isActive, isSuccessfullyFinished, renderName } from "@/utils/crawler"; + +type Crawls = APIPaginatedList; +const SORT_DIRECTIONS = ["asc", "desc"] as const; +type SortDirection = (typeof SORT_DIRECTIONS)[number]; + +type Keys = (keyof T)[]; + +const POLL_INTERVAL_SECONDS = 5; +const INITIAL_PAGE_SIZE = 20; +const FILTER_BY_CURRENT_USER_STORAGE_KEY = "btrix.filterByCurrentUser.crawls"; +const sortableFields: Record< + string, + { label: string; defaultDirection?: SortDirection } +> = { + started: { + label: msg("Date Started"), + defaultDirection: "desc", + }, + finished: { + label: msg("Date Finished"), + defaultDirection: "desc", + }, + crawlExecSeconds: { + label: msg("Execution Time"), + defaultDirection: "desc", + }, + pageCount: { + label: msg("Pages"), + defaultDirection: "desc", + }, + fileSize: { + label: msg("Size"), + defaultDirection: "desc", + }, +}; + +type SortField = keyof typeof sortableFields; + +type SortBy = { + field: SortField; + direction: SortDirection; +}; +const DEFAULT_SORT_BY: SortBy = { + field: "started", + direction: sortableFields["started"].defaultDirection!, +}; + +type FilterBy = { + name?: string; + firstSeed?: string; + state?: CrawlState[]; +}; + +/** + * A crawl is the operation of a workflow. This is distinct from an archived item, + * which is a crawl that has successfully finished and can be replayed. Users can + * view, search, and filter crawls from this page. + */ +@customElement("btrix-org-crawls") +@localized() +export class OrgCrawls extends BtrixElement { + @state() + private pagination: Required = { + page: parsePage(new URLSearchParams(location.search).get("page")), + pageSize: INITIAL_PAGE_SIZE, + }; + + @query("btrix-pagination") + private readonly paginationElement?: Pagination; + + @state() + private searchOptions: Record[] = []; + + @state() + private readonly orderBy = new SearchParamsValue( + this, + (value, params) => { + if (value.field === DEFAULT_SORT_BY.field) { + params.delete("sortBy"); + } else { + params.set("sortBy", value.field); + } + if (value.direction === sortableFields[value.field].defaultDirection) { + params.delete("sortDir"); + } else { + params.set("sortDir", value.direction); + } + return params; + }, + (params) => { + const field = params.get("sortBy"); + if (!field) { + return DEFAULT_SORT_BY; + } + let direction = params.get("sortDir"); + if ( + !direction || + (SORT_DIRECTIONS as readonly string[]).includes(direction) + ) { + direction = + sortableFields[field].defaultDirection || DEFAULT_SORT_BY.direction; + } + return { field, direction: direction as SortDirection }; + }, + ); + private readonly filterByCurrentUser = new SearchParamsValue( + this, + (value, params) => { + if (value) { + params.set("mine", "true"); + } else { + params.delete("mine"); + } + return params; + }, + (params) => params.get("mine") === "true", + { + initial: (initialValue) => + window.sessionStorage.getItem(FILTER_BY_CURRENT_USER_STORAGE_KEY) === + "true" || + initialValue || + false, + }, + ); + private readonly filterByTags = new SearchParamsValue( + this, + (value, params) => { + params.delete("tags"); + value?.forEach((v) => { + params.append("tags", v); + }); + return params; + }, + (params) => params.getAll("tags"), + ); + + private readonly filterByTagsType = new SearchParamsValue<"and" | "or">( + this, + (value, params) => { + if (value === "and") { + params.set("tagsType", value); + } else { + params.delete("tagsType"); + } + return params; + }, + (params) => (params.get("tagsType") === "and" ? "and" : "or"), + ); + + private readonly filterBy = new SearchParamsValue( + this, + (value, params) => { + const keys = ["name", "firstSeed", "state"] as (keyof FilterBy)[]; + keys.forEach((key) => { + if (value[key] == null) { + params.delete(key); + } else { + switch (key) { + case "firstSeed": + case "name": + params.set(key, value[key]); + break; + case "state": + params.delete("status"); + value[key].forEach((state) => { + params.append("status", state); + }); + break; + } + } + }); + return params; + }, + (params) => { + const state = params.getAll("status") as CrawlState[]; + + return { + name: params.get("name") ?? undefined, + firstSeed: params.get("firstSeed") ?? undefined, + state: state.length ? state : undefined, + }; + }, + ); + + @state() + private crawlToEdit: Crawl | null = null; + + @state() + private isEditingItem = false; + + @state() + private crawlToDelete: Crawl | null = null; + + @state() + private isDeletingItem = false; + + @query("#stateSelect") + stateSelect?: SlSelect; + + private get hasFiltersSet() { + return [ + this.filterBy.value.firstSeed, + this.filterBy.value.name, + this.filterBy.value.state?.length || undefined, + this.filterByCurrentUser.value || undefined, + this.filterByTags.value?.length || undefined, + ].some((v) => v !== undefined); + } + + private clearFilters() { + this.filterBy.setValue({ + ...this.filterBy.value, + firstSeed: undefined, + name: undefined, + state: undefined, + }); + this.filterByCurrentUser.setValue(false); + this.filterByTags.setValue(undefined); + } + + private readonly crawlsTask = new Task(this, { + task: async ( + [ + pagination, + orderBy, + filterBy, + filterByCurrentUser, + filterByTags, + filterByTagsType, + ], + { signal }, + ) => { + try { + const data = await this.getCrawls( + { + pagination, + orderBy, + filterBy, + filterByCurrentUser, + filterByTags, + filterByTagsType, + }, + signal, + ); + + if (this.getArchivedItemsTimeout) { + window.clearTimeout(this.getArchivedItemsTimeout); + } + + this.getArchivedItemsTimeout = window.setTimeout(() => { + void this.crawlsTask.run(); + }, POLL_INTERVAL_SECONDS * 1000); + + return data; + } catch (e) { + if ((e as Error).name === "AbortError") { + console.debug("Fetch crawls aborted to throttle"); + } else { + this.notify.toast({ + message: msg("Sorry, couldn’t retrieve crawl runs at this time."), + variant: "danger", + icon: "exclamation-octagon", + id: "crawl-status", + }); + } + throw e; + } + }, + args: () => + // TODO consolidate filters into single fetch params + [ + this.pagination, + this.orderBy.value, + this.filterBy.value, + this.filterByCurrentUser.value, + this.filterByTags.value, + this.filterByTagsType.value, + ] as const, + }); + + private getArchivedItemsTimeout?: number; + + // For fuzzy search: + private readonly searchKeys = ["name", "firstSeed"]; + + private get selectedSearchFilterKey() { + return ( + Object.keys(WorkflowSearch.FieldLabels) as Keys< + typeof WorkflowSearch.FieldLabels + > + ).find((key) => Boolean(this.filterBy.value[key])); + } + + constructor() { + super(); + this.filterByCurrentUser.setValue( + window.sessionStorage.getItem(FILTER_BY_CURRENT_USER_STORAGE_KEY) === + "true", + ); + } + + protected willUpdate( + changedProperties: PropertyValues & Map, + ) { + if ( + changedProperties.has("filterByCurrentUser.value") || + changedProperties.has("filterBy.value") || + changedProperties.has("orderBy.value") || + changedProperties.has("filterByTags.value") || + changedProperties.has("filterByTagsType.value") + ) { + this.paginationElement?.setPage(1, { dispatch: true, replace: true }); + + if (changedProperties.has("filterByCurrentUser")) { + window.sessionStorage.setItem( + FILTER_BY_CURRENT_USER_STORAGE_KEY, + this.filterByCurrentUser.value.toString(), + ); + } + } + } + + protected firstUpdated() { + void this.fetchConfigSearchValues(); + } + + disconnectedCallback(): void { + window.clearTimeout(this.getArchivedItemsTimeout); + super.disconnectedCallback(); + } + + render() { + return html` +
+
+ ${this.renderControls()} +
+ + ${this.crawlsTask.render({ + initial: () => html` +
+ +
+ `, + pending: () => + // TODO differentiate between pending between poll and + // pending from user action, in order to show loading indicator + this.crawlsTask.value + ? // Render previous value while latest is loading + this.renderArchivedItems(this.crawlsTask.value) + : nothing, + complete: this.renderArchivedItems, + })} +
+ `; + } + + private readonly renderArchivedItems = ({ + items, + page, + total, + pageSize, + }: APIPaginatedList) => html` +
+ ${items.length + ? html` + + + ${msg("Row actions")} + + ${repeat(items, ({ id }) => id, this.renderArchivedItem)} + + ` + : this.renderEmptyState()} +
+ ${when( + total > pageSize, + () => html` +
+ { + this.pagination = { + ...this.pagination, + page: e.detail.page, + }; + await this.updateComplete; + + // Scroll to top of list + // TODO once deep-linking is implemented, scroll to top of pushstate + this.scrollIntoView({ behavior: "smooth" }); + }} + > +
+ `, + )} + ${this.crawlToEdit + ? html` + (this.isEditingItem = false)} + @updated=${() => { + /* TODO fetch current page or single crawl */ + void this.crawlsTask.run(); + }} + > + ` + : nothing} + + (this.isDeletingItem = false)} + @btrix-confirm=${() => { + this.isDeletingItem = false; + if (this.crawlToDelete) { + void this.deleteItem(this.crawlToDelete); + } + }} + > + ${this.crawlToDelete?.finished + ? html`${renderName(this.crawlToDelete)} + (${this.localize.date(this.crawlToDelete.finished)})` + : nothing} + + `; + + private renderControls() { + return html` +
+
${this.renderSearch()}
+ +
+ + ${this.renderSortControl()} +
+
+ + ${msg("Filter by:")} + + { + this.filterBy.setValue({ + ...this.filterBy.value, + state: e.detail.value, + }); + }} + > + + ${this.userInfo?.id + ? html` { + const { checked } = e.target as FilterChip; + this.filterByCurrentUser.setValue(Boolean(checked)); + }} + > + ${msg("Mine")} + ` + : ""} + ${when( + this.hasFiltersSet, + () => html` + + + ${msg("Clear All")} + + `, + )} +
+
+ `; + } + + private renderSortControl() { + const options = Object.entries(sortableFields).map( + ([value, { label }]) => html` + ${label} + `, + ); + return html` + { + const field = (e.target as HTMLSelectElement).value; + this.orderBy.setValue({ + field: field, + direction: + sortableFields[field].defaultDirection || + this.orderBy.value.direction, + }); + }} + > + ${options} + + + { + this.orderBy.setValue({ + ...this.orderBy.value, + direction: + this.orderBy.value.direction === "asc" ? "desc" : "asc", + }); + }} + > + + `; + } + + private renderSearch() { + return html` + ) => { + const { key, value } = e.detail; + console.log(key, value); + if (key == null) return; + this.filterBy.setValue({ + ...this.filterBy.value, + [key]: value, + }); + }} + @btrix-clear=${() => { + const { + name: _name, + firstSeed: _firstSeed, + ...otherFilters + } = this.filterBy.value; + this.filterBy.setValue(otherFilters); + }} + > + + `; + } + + private readonly renderArchivedItem = (crawl: Crawl) => html` + + ${this.renderMenuItems(crawl)} + + `; + + private readonly renderMenuItems = (crawl: Crawl) => { + const authToken = this.authState?.headers.Authorization.split(" ")[1]; + const isCrawler = this.appState.isCrawler; + const isSuccess = isSuccessfullyFinished(crawl); + + return html` + ${when( + isCrawler, + () => html` + { + this.crawlToEdit = crawl; + await this.updateComplete; + this.isEditingItem = true; + }} + > + + ${isSuccess ? msg("Edit Archived Item") : msg("Edit Metadata")} + + + `, + )} + ${when( + isSuccess, + () => html` + + + ${msg("Download Item")} + ${crawl.fileSize + ? html` ${this.localize.bytes(crawl.fileSize)}` + : nothing} + + + `, + )} + + this.navigate.to( + `${this.navigate.orgBasePath}/workflows/${crawl.cid}`, + )} + > + + ${msg("Go to Workflow")} + + ClipboardController.copyToClipboard(crawl.cid)} + > + + ${msg("Copy Workflow ID")} + + + + ClipboardController.copyToClipboard(crawl.tags.join(", "))} + ?disabled=${!crawl.tags.length} + > + + ${msg("Copy Tags")} + + ClipboardController.copyToClipboard(crawl.id)} + > + + ${msg("Copy Crawl ID")} + + ${when( + isCrawler && !isActive(crawl), + () => html` + + { + if (isSuccess) { + this.confirmDeleteItem(crawl); + } else { + void this.deleteItem(crawl); + } + }} + > + + ${msg("Delete Crawl")} + + `, + )} + `; + }; + + private renderEmptyState() { + if (this.hasFiltersSet) { + return html` +
+

+ ${msg("No matching items found.")} + +

+
+ `; + } + + if (this.pagination.page && this.pagination.page > 1) { + return html` +
+

+ ${msg("Could not find page.")} +

+
+ `; + } + + return html` +
+

${msg("No crawl runs yet.")}

+
+ `; + } + + private async getCrawls( + params: { + pagination: OrgCrawls["pagination"]; + orderBy: OrgCrawls["orderBy"]["value"]; + filterBy: OrgCrawls["filterBy"]["value"]; + filterByCurrentUser: OrgCrawls["filterByCurrentUser"]["value"]; + filterByTags: OrgCrawls["filterByTags"]["value"]; + filterByTagsType: OrgCrawls["filterByTagsType"]["value"]; + }, + signal: AbortSignal, + ) { + const query = queryString.stringify( + { + ...params.filterBy, + state: params.filterBy.state, + page: params.pagination.page, + pageSize: params.pagination.pageSize, + tags: params.filterByTags, + tagMatch: params.filterByTags?.length + ? params.filterByTagsType + : undefined, + userid: params.filterByCurrentUser ? this.userInfo!.id : undefined, + sortBy: params.orderBy.field, + sortDirection: params.orderBy.direction === "desc" ? -1 : 1, + }, + { + arrayFormat: "none", + }, + ); + + return this.api.fetch(`/orgs/${this.orgId}/crawls?${query}`, { + signal, + }); + } + + private async fetchConfigSearchValues() { + try { + const query = queryString.stringify({ + crawlType: "crawl", + }); + const data: { + crawlIds: string[]; + names: string[]; + descriptions: string[]; + firstSeeds: string[]; + } = await this.api.fetch( + `/orgs/${this.orgId}/all-crawls/search-values?${query}`, + ); + + // Update search/filter collection + const toSearchItem = (key: SearchFields) => (value: string) => ({ + [key]: value, + }); + this.searchOptions = [ + ...data.names.map(toSearchItem("name")), + ...data.firstSeeds.map(toSearchItem("firstSeed")), + ]; + } catch (e) { + console.debug(e); + } + } + + private readonly confirmDeleteItem = (item: Crawl) => { + this.crawlToDelete = item; + this.isDeletingItem = true; + }; + + private async deleteItem(item: Crawl) { + try { + const _data = await this.api.fetch(`/orgs/${item.oid}/crawls/delete`, { + method: "POST", + body: JSON.stringify({ + crawl_ids: [item.id], + }), + }); + // TODO eager list update before server response + void this.crawlsTask.run(); + // const { items, ...crawlsData } = this.archivedItems!; + this.crawlToDelete = null; + // this.archivedItems = { + // ...crawlsData, + // items: items.filter((c) => c.id !== item.id), + // }; + this.notify.toast({ + message: msg(str`Successfully deleted crawl.`), + variant: "success", + icon: "check2-circle", + id: "crawl-status", + }); + } catch (e) { + if (this.crawlToDelete) { + this.confirmDeleteItem(this.crawlToDelete); + } + let message = msg(str`Sorry, couldn't delete crawl at this time.`); + if (isApiError(e)) { + if (e.details == "not_allowed") { + message = msg(str`Only org owners can delete other users' crawls.`); + } else if (e.message) { + message = e.message; + } + } + this.notify.toast({ + message: message, + variant: "danger", + icon: "exclamation-octagon", + id: "crawl-status", + }); + } + } + + async getWorkflow(crawl: Crawl): Promise { + const data: Workflow = await this.api.fetch( + `/orgs/${crawl.oid}/crawlconfigs/${crawl.cid}`, + ); + + return data; + } +} diff --git a/frontend/src/pages/org/index.ts b/frontend/src/pages/org/index.ts index 7fd5f19c88..83bfc492fb 100644 --- a/frontend/src/pages/org/index.ts +++ b/frontend/src/pages/org/index.ts @@ -24,7 +24,7 @@ import type { QuotaUpdateDetail } from "@/controllers/api"; import needLogin from "@/decorators/needLogin"; import type { CollectionSavedEvent } from "@/features/collections/collection-create-dialog"; import type { SelectJobTypeEvent } from "@/features/crawl-workflows/new-workflow-dialog"; -import { OrgTab, RouteNamespace, WorkflowTab } from "@/routes"; +import { CommonTab, OrgTab, RouteNamespace, WorkflowTab } from "@/routes"; import type { ProxiesAPIResponse } from "@/types/crawler"; import type { UserOrg } from "@/types/user"; import { isApiError } from "@/utils/api"; @@ -35,7 +35,9 @@ import { type OrgData } from "@/utils/orgs"; import { AppStateService } from "@/utils/state"; import type { FormState as WorkflowFormState } from "@/utils/workflow"; +import "./crawling"; import "./workflow-detail"; +import "./crawls"; import "./workflows-list"; import "./archived-item-detail"; import "./archived-items"; @@ -548,7 +550,13 @@ export class Org extends BtrixElement { `; } - if (this.orgPath.startsWith("/workflows/new")) { + const crawlingTab = this.orgPath + .slice(this.orgPath.indexOf(OrgTab.Workflows) + OrgTab.Workflows.length) + .replace(/(^\/|\/$)/, "") + .split("/")[0] + .split("?")[0]; + + if ((crawlingTab as CommonTab) === CommonTab.New) { const { workflow, seeds, seedFile, scopeType } = (this.viewStateData || {}) satisfies Partial; @@ -563,7 +571,8 @@ export class Org extends BtrixElement { >`; } - return html` { this.openDialogName = undefined; @@ -578,7 +587,7 @@ export class Org extends BtrixElement { scopeType: e.detail, }); }} - >`; + >`; }; private readonly renderBrowserProfiles = () => { diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 4cae60afab..8b836846db 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -1,6 +1,6 @@ import { localized, msg, str } from "@lit/localize"; import { Task, TaskStatus } from "@lit/task"; -import type { SlDropdown, SlSelect } from "@shoelace-style/shoelace"; +import type { SlDropdown } from "@shoelace-style/shoelace"; import clsx from "clsx"; import { html, nothing, type PropertyValues, type TemplateResult } from "lit"; import { customElement, property, query, state } from "lit/decorators.js"; @@ -17,12 +17,9 @@ import type { Crawl, CrawlLog, Seed, Workflow } from "./types"; import { BtrixElement } from "@/classes/BtrixElement"; import type { Alert } from "@/components/ui/alert"; -import { - calculatePages, - parsePage, - type PageChangeEvent, -} from "@/components/ui/pagination"; +import { parsePage, type PageChangeEvent } from "@/components/ui/pagination"; import { ClipboardController } from "@/controllers/clipboard"; +import type { BtrixChangeArchivedItemStateFilterEvent } from "@/features/archived-items/archived-item-state-filter"; import { CrawlStatus } from "@/features/archived-items/crawl-status"; import { ExclusionEditor } from "@/features/crawl-workflows/exclusion-editor"; import { ShareableNotice } from "@/features/crawl-workflows/templates/shareable-notice"; @@ -41,7 +38,6 @@ import { isApiError } from "@/utils/api"; import { settingsForDuplicate } from "@/utils/crawl-workflows/settingsForDuplicate"; import { DEFAULT_MAX_SCALE, - inactiveCrawlStates, isActive, isSkipped, isSuccessfullyFinished, @@ -565,36 +561,27 @@ export class WorkflowDetail extends BtrixElement { > - (this.openDialogName = undefined)} - @sl-show=${this.showDialog} + (this.openDialogName = undefined)} @sl-after-hide=${() => (this.isDialogVisible = false)} + @btrix-confirm=${() => { + this.openDialogName = undefined; + if (this.crawlToDelete) { + void this.deleteCrawl(this.crawlToDelete); + } + }} > - ${msg( - "All files and logs associated with this crawl will also be deleted, and the crawl will be removed from any Collection it is a part of.", - )} -
- (this.openDialogName = undefined)} - >${msg("Cancel")} - { - this.openDialogName = undefined; - if (this.crawlToDelete) { - await this.deleteCrawl(this.crawlToDelete); - } - }} - >${msg("Delete Crawl")} -
- + ${this.crawlToDelete?.finished + ? html`${msg("crawl that finished on")} + ${this.localize.date(this.crawlToDelete.finished)}` + : nothing} + ) => { - const pages = calculatePages(crawls); - - if (crawls.page === 1 || pages < 2) return; - - const page = this.localize.number(crawls.page); - const pageCount = this.localize.number(pages); - - return msg(str`Viewing page ${page} of ${pageCount}`); - }; - return html`
-
${when(this.crawls, pageView)}
-
-
${msg("Status:")}
- { - const value = (e.target as SlSelect).value as CrawlState[]; - await this.updateComplete; +
+ + ${msg("Filter by:")} + + { this.crawlsParams = { ...this.crawlsParams, page: 1, - state: value, + state: e.detail.value, }; }} - > - ${inactiveCrawlStates.map(this.renderStatusMenuItem)} - + > + ${when( + this.crawlsParams.state?.length, + () => html` + { + this.crawlsParams = { + ...this.crawlsParams, + state: undefined, + }; + }} + > + + ${msg("Clear")} + + `, + )}
@@ -1232,7 +1219,7 @@ export class WorkflowDetail extends BtrixElement { () => html` this.confirmDeleteCrawl(crawl)} > @@ -1285,7 +1272,7 @@ export class WorkflowDetail extends BtrixElement { : html`

- ${this.crawls?.total + ${this.crawlsParams.state ? msg("No matching crawls found.") : msg("No crawls yet.")}

@@ -2092,7 +2079,7 @@ export class WorkflowDetail extends BtrixElement { cid: workflowId, sortBy: "started", page: params.page ?? this.crawls?.page, - pageSize: this.crawls?.pageSize ?? 10, + pageSize: this.crawls?.pageSize ?? 50, ...params, }, { diff --git a/frontend/src/pages/org/workflows-list.ts b/frontend/src/pages/org/workflows-list.ts index 19898f49cc..9d80600a62 100644 --- a/frontend/src/pages/org/workflows-list.ts +++ b/frontend/src/pages/org/workflows-list.ts @@ -1,6 +1,6 @@ import { localized, msg, str } from "@lit/localize"; import { Task } from "@lit/task"; -import type { SlDialog, SlSelectEvent } from "@shoelace-style/shoelace"; +import type { SlDialog } from "@shoelace-style/shoelace"; import clsx from "clsx"; import { html, type PropertyValues } from "lit"; import { customElement, query, state } from "lit/decorators.js"; @@ -8,12 +8,7 @@ import { ifDefined } from "lit/directives/if-defined.js"; import { when } from "lit/directives/when.js"; import queryString from "query-string"; -import { - ScopeType, - type ListWorkflow, - type Seed, - type Workflow, -} from "./types"; +import { type ListWorkflow, type Seed, type Workflow } from "./types"; import { BtrixElement } from "@/classes/BtrixElement"; import type { @@ -27,7 +22,6 @@ import { } from "@/components/ui/pagination"; import { type SelectEvent } from "@/components/ui/search-combobox"; import { SearchParamsValue } from "@/controllers/searchParamsValue"; -import type { SelectJobTypeEvent } from "@/features/crawl-workflows/new-workflow-dialog"; import { Action, type BtrixSelectActionEvent, @@ -35,24 +29,22 @@ import { import { type BtrixChangeWorkflowLastCrawlStateFilterEvent } from "@/features/crawl-workflows/workflow-last-crawl-state-filter"; import { type BtrixChangeWorkflowProfileFilterEvent } from "@/features/crawl-workflows/workflow-profile-filter"; import type { BtrixChangeWorkflowScheduleFilterEvent } from "@/features/crawl-workflows/workflow-schedule-filter"; +import { + WorkflowSearch, + type SearchFields, +} from "@/features/crawl-workflows/workflow-search"; import type { BtrixChangeWorkflowTagFilterEvent } from "@/features/crawl-workflows/workflow-tag-filter"; -import { pageHeader } from "@/layouts/pageHeader"; import { WorkflowTab } from "@/routes"; -import scopeTypeLabels from "@/strings/crawl-workflows/scopeType"; import { deleteConfirmation } from "@/strings/ui"; import type { APIPaginatedList, APIPaginationQuery } from "@/types/api"; import { type CrawlState } from "@/types/crawlState"; -import { - NewWorkflowOnlyScopeType, - type StorageSeedFile, -} from "@/types/workflow"; +import { type StorageSeedFile } from "@/types/workflow"; import { isApiError } from "@/utils/api"; import { settingsForDuplicate } from "@/utils/crawl-workflows/settingsForDuplicate"; import { renderName } from "@/utils/crawler"; import { isNotEqual } from "@/utils/is-not-equal"; import { tw } from "@/utils/tailwind"; -type SearchFields = "name" | "firstSeed"; type SortField = "lastRun" | "name" | "firstSeed" | "created" | "modified"; const SORT_DIRECTIONS = ["asc", "desc"] as const; type SortDirection = (typeof SORT_DIRECTIONS)[number]; @@ -116,11 +108,6 @@ type FilterBy = { @customElement("btrix-workflows-list") @localized() export class WorkflowsList extends BtrixElement { - static FieldLabels: Record = { - name: msg("Name"), - firstSeed: msg("Crawl Start URL"), - }; - @state({ hasChanged: isNotEqual }) private pagination: Required = { page: parsePage(new URLSearchParams(location.search).get("page")), @@ -285,13 +272,10 @@ export class WorkflowsList extends BtrixElement { @query("#deleteDialog") private readonly deleteDialog?: SlDialog | null; - // For fuzzy search: - private readonly searchKeys = ["name", "firstSeed"]; - private get selectedSearchFilterKey() { return ( - Object.keys(WorkflowsList.FieldLabels) as Keys< - typeof WorkflowsList.FieldLabels + Object.keys(WorkflowSearch.FieldLabels) as Keys< + typeof WorkflowSearch.FieldLabels > ).find((key) => Boolean(this.filterBy.value[key])); } @@ -416,108 +400,8 @@ export class WorkflowsList extends BtrixElement { render() { return html` -
- ${pageHeader({ - title: msg("Crawl Workflows"), - actions: html` - ${when( - this.appState.isAdmin, - () => - html` - - `, - )} - ${when( - this.appState.isCrawler, - () => html` - - - this.navigate.to( - `${this.navigate.orgBasePath}/workflows/new`, - { - scopeType: - this.appState.userPreferences?.newWorkflowScopeType, - }, - )} - > - - ${msg("New Workflow")} - { - const { value } = e.detail.item; - - if (value) { - this.dispatchEvent( - new CustomEvent( - "select-job-type", - { - detail: value as SelectJobTypeEvent["detail"], - }, - ), - ); - } - }} - > - - ${msg("Scope options")} - - - ${msg("Page Crawl")} - ${scopeTypeLabels[ScopeType.Page]} - - ${scopeTypeLabels[NewWorkflowOnlyScopeType.PageList]} - - - ${scopeTypeLabels[ScopeType.SPA]} - - - ${msg("Site Crawl")} - - ${scopeTypeLabels[ScopeType.Prefix]} - - - ${scopeTypeLabels[ScopeType.Host]} - - - ${scopeTypeLabels[ScopeType.Domain]} - - - ${scopeTypeLabels[ScopeType.Custom]} - - - - - `, - )} - `, - classNames: tw`border-b-transparent`, - })} -
- ${this.renderControls()} -
+
+ ${this.renderControls()}
${when( @@ -739,17 +623,14 @@ export class WorkflowsList extends BtrixElement { private renderSearch() { return html` - ) => { + @btrix-select=${(e: SelectEvent) => { const { key, value } = e.detail; if (key == null) return; this.filterBy.setValue({ @@ -766,7 +647,7 @@ export class WorkflowsList extends BtrixElement { this.filterBy.setValue(otherFilters); }} > - + `; } diff --git a/frontend/src/pages/org/workflows-new.ts b/frontend/src/pages/org/workflows-new.ts index 6c6692fa22..4f706f2c7b 100644 --- a/frontend/src/pages/org/workflows-new.ts +++ b/frontend/src/pages/org/workflows-new.ts @@ -81,7 +81,7 @@ export class WorkflowsNew extends BtrixElement { const breadcrumbs: Breadcrumb[] = [ { href: `${this.navigate.orgBasePath}/workflows`, - content: msg("Crawl Workflows"), + content: msg("Crawling"), }, { content: msg("New Workflow"), diff --git a/frontend/src/routes.ts b/frontend/src/routes.ts index 5ad69d0fba..3dabc5eda2 100644 --- a/frontend/src/routes.ts +++ b/frontend/src/routes.ts @@ -20,6 +20,11 @@ export enum WorkflowTab { Settings = "settings", } +export enum CommonTab { + New = "new", + View = "view", +} + const archivedItemPath = "/:itemId(/review/:qaTab)"; export const ROUTES = { @@ -38,9 +43,9 @@ export const ROUTES = { `/${RouteNamespace.PrivateOrgs}/:slug(/)`, // Org sections: `(/${OrgTab.Dashboard})`, - `(/${OrgTab.Workflows}(/new)(/:workflowId(/:workflowTab)(/crawls${archivedItemPath})))`, + `(/${OrgTab.Workflows}(/${WorkflowTab.Crawls})(/${CommonTab.New})(/:workflowId(/:workflowTab)(/${WorkflowTab.Crawls}${archivedItemPath})))`, `(/${OrgTab.Items}(/:itemType(${archivedItemPath})))`, - `(/${OrgTab.Collections}(/new)(/view/:collectionId(/:collectionTab)))`, + `(/${OrgTab.Collections}(/${CommonTab.New})(/${CommonTab.View}/:collectionId(/:collectionTab)))`, `(/${OrgTab.BrowserProfiles}(/profile(/browser/:browserId)(/:browserProfileId)))`, `(/${OrgTab.Settings}(/:settingsTab))`, ].join(""), diff --git a/frontend/src/stories/design/status-indicators.mdx b/frontend/src/stories/design/status-indicators.mdx index e23fd05153..8be5e727f2 100644 --- a/frontend/src/stories/design/status-indicators.mdx +++ b/frontend/src/stories/design/status-indicators.mdx @@ -17,7 +17,7 @@ buttons or actions which use strokes. When used without labels, status icons should include tooltips to provider further clarity as to what they indicate. -## In Use (as of 2025-05-06) +## In Use (as of 2025-10-21) | Icon & Label | Color | Context | Description | | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | --------------- | --------------------------------------------------------------- | @@ -26,14 +26,15 @@ further clarity as to what they indicate. | 1 Crawl Workflow Waiting | Shoelace `--sl-color-violet-600` | Dashboard | Count of crawls in "waiting" status | | No Crawls Yet | Shoelace `--sl-color-neutral-400` | Crawl Workflows | Used to show that a workflow has no crawls | | Complete | Shoelace `--sl-color-success-600` | Crawl Workflows | Used to show that a workflow's most recent crawl was completed | -| Stopped | Shoelace `--sl-color-warning-600` | Crawl Workflows | Used to show that a workflow's most recent crawl was stopped | -| Canceled | Shoelace `--sl-color-danger-600` | Crawl Workflows | Used to show that a workflow's most recent crawl was canceled | -| Starting | Shoelace `--sl-color-violet-600` | Crawl Workflows | Used to show that a crawl is starting | +| Stopped | Shoelace `--sl-color-warning-600` | Crawl Workflows | Used to show that a workflow's most recent crawl was stopped by the user | +| Stopped: [Reason] | Shoelace `--sl-color-warning-600` | Crawl Workflows | Used to show that a workflow's most recent crawl was stopped due to system reason | +| Canceled | Shoelace `--sl-color-neutral-600` | Crawl Workflows | Used to show that a workflow's most recent crawl was canceled | +| Starting | Shoelace `--sl-color-violet-600` | Crawl Workflows | Used to show that a crawl is starting | | Running | Shoelace `--sl-color-success-600` | Crawl Workflows | Used to show that a crawl is running | | Behavior timed out | Shoelace `--sl-color-warning-600` | Crawl Logs | Used to show a warning log from a behavior | | Success | Shoelace `--sl-color-success-600` | Toasts | Used to show a success notification | | Warning | Shoelace `--sl-color-warning-600` | Toasts | Used to show a warning notification | -| Danger | Shoelace `--sl-color-danger-600` | Toasts | Used to show an error notification | +| Danger | Shoelace `--sl-color-danger-600` | Toasts | Used to show an error notification | ## Intended Implementation @@ -48,4 +49,4 @@ further clarity as to what they indicate. | Incomplete | Shoelace `--sl-color-warning-600` | Used for states that are ambiguous or partially satisfied, but no longer running | `dash-square-fill` | Stopped | | Warning | Shoelace `--sl-color-warning-600` | Used for warning states, something is wrong but not critically | `exclamation-diamond-fill` or `exclamation-diamond` | Warning | | Danger | Shoelace `--sl-color-danger-600` | Used for non-fatal errors that may be addressed by the user | `exclamation-triangle-fill` or `exclamation-triangle` | Payment Failed | -| Fatal | Shoelace `--sl-color-danger-600` | Used for fatal errors and actions that result in data loss | `x-octagon-fill` or `x-octagon` | Canceled | +| Fatal | Shoelace `--sl-color-danger-600` | Used for fatal errors and actions that result in data loss | `x-octagon-fill` or `x-octagon` | Cancel | diff --git a/frontend/src/utils/pluralize.ts b/frontend/src/utils/pluralize.ts index dcfe3fe369..08862130c8 100644 --- a/frontend/src/utils/pluralize.ts +++ b/frontend/src/utils/pluralize.ts @@ -247,6 +247,32 @@ const plurals = { id: "profiles.plural.other", }), }, + collections: { + zero: msg("collections", { + desc: 'plural form of "collections" for zero collections', + id: "collections.plural.zero", + }), + one: msg("collection", { + desc: 'singular form for "collection"', + id: "collections.plural.one", + }), + two: msg("collections", { + desc: 'plural form of "collections" for two collections', + id: "collections.plural.two", + }), + few: msg("collections", { + desc: 'plural form of "collections" for few collections', + id: "collections.plural.few", + }), + many: msg("collections", { + desc: 'plural form of "collections" for many collections', + id: "collections.plural.many", + }), + other: msg("collections", { + desc: 'plural form of "collections" for multiple/other collections', + id: "collections.plural.other", + }), + }, changes: { zero: msg("changes", { desc: 'plural form of "changes" for zero changes',