From 0b49bc88f6e30701dfde95106e5517e499dae5fb Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 28 Oct 2025 11:44:14 +0100 Subject: [PATCH 01/15] WIP add usedStorageBytes to dataset compact writes --- app/models/dataset/Dataset.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/models/dataset/Dataset.scala b/app/models/dataset/Dataset.scala index a10dc03c31a..c0b8f56d60b 100755 --- a/app/models/dataset/Dataset.scala +++ b/app/models/dataset/Dataset.scala @@ -89,6 +89,7 @@ case class DatasetCompactInfo( isUnreported: Boolean, colorLayerNames: List[String], segmentationLayerNames: List[String], + usedStorageBytes: Option[Long], ) { def dataSourceId = new DataSourceId(directoryName, owningOrganization) } @@ -288,7 +289,8 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA d.status, d.tags, cl.names AS colorLayerNames, - sl.names AS segmentationLayerNames + sl.names AS segmentationLayerNames, + 0 AS usedStorageBytes -- // TODO FROM (SELECT $columns FROM $existingCollectionName WHERE $selectionPredicates $limitQuery) d JOIN webknossos.organizations o @@ -316,7 +318,8 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA String, String, String, - String)]) + String, + Option[Long])]) } yield rows.toList.map( row => @@ -334,7 +337,8 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA tags = parseArrayLiteral(row._11), isUnreported = DataSourceStatus.unreportedStatusList.contains(row._10), colorLayerNames = parseArrayLiteral(row._12), - segmentationLayerNames = parseArrayLiteral(row._13) + segmentationLayerNames = parseArrayLiteral(row._13), + usedStorageBytes = row._14, )) private def buildSelectionPredicates(isActiveOpt: Option[Boolean], From af886eca278d48dd5f4355d648f53c2e537002d8 Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Tue, 28 Oct 2025 16:57:25 +0100 Subject: [PATCH 02/15] basic frontend to show dataset storage size --- .../advanced_dataset/dataset_table.tsx | 25 +++++++++++++++++-- frontend/javascripts/types/api_types.ts | 2 ++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx index 252ea6fba47..b47e6666205 100644 --- a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx +++ b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx @@ -22,7 +22,7 @@ import { useDatasetDrop, } from "dashboard/folders/folder_tree"; import { diceCoefficient as dice } from "dice-coefficient"; -import { stringToColor } from "libs/format_utils"; +import { formatCountToDataAmountUnit, stringToColor } from "libs/format_utils"; import { useWkSelector } from "libs/react_hooks"; import Shortcut from "libs/shortcut_component"; import * as Utils from "libs/utils"; @@ -287,6 +287,11 @@ class DatasetRenderer { return DatasetRenderer.getRowKey(this.data); } + renderStorageColumn(): React.ReactNode { + return this.data.usedStorageBytes != null + ? formatCountToDataAmountUnit(this.data.usedStorageBytes, true) + : null; + } renderTypeColumn(): React.ReactNode { return ; } @@ -388,6 +393,9 @@ class FolderRenderer { ); } + renderStorageColumn(): React.ReactNode { + return null; + } renderCreationDateColumn(): React.ReactNode { return null; } @@ -604,7 +612,20 @@ class DatasetTable extends React.PureComponent { sortOrder: sortedInfo.columnKey === "created" ? sortedInfo.order : undefined, render: (_created, rowRenderer: RowRenderer) => rowRenderer.renderCreationDateColumn(), }, - + { + title: "Size", + key: "size", + width: 120, + render: (_: any, rowRenderer: RowRenderer) => { + return isRecordADataset(rowRenderer.data) ? rowRenderer.renderStorageColumn() : null; + }, + sorter: Utils.compareBy((rowRenderer) => + isRecordADataset(rowRenderer.data) && rowRenderer.data.usedStorageBytes + ? rowRenderer.data.usedStorageBytes + : 0, + ), + sortOrder: sortedInfo.columnKey === "storage" ? sortedInfo.order : undefined, + }, { width: 200, title: "Actions", diff --git a/frontend/javascripts/types/api_types.ts b/frontend/javascripts/types/api_types.ts index 20f958467c8..856c28841d2 100644 --- a/frontend/javascripts/types/api_types.ts +++ b/frontend/javascripts/types/api_types.ts @@ -266,6 +266,7 @@ export type APIDatasetCompactWithoutStatusAndLayerNames = Pick< | "lastUsedByUser" | "tags" | "isUnreported" + | "usedStorageBytes" >; export type APIDatasetCompact = APIDatasetCompactWithoutStatusAndLayerNames & { id: string; @@ -295,6 +296,7 @@ export function convertDatasetToCompact(dataset: APIDataset): APIDatasetCompact isUnreported: dataset.isUnreported, colorLayerNames: colorLayerNames, segmentationLayerNames: segmentationLayerNames, + usedStorageBytes: dataset.usedStorageBytes, }; } From 9d09e459384089d1cdb7d23164eccca27cfc748a Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 29 Oct 2025 13:47:23 +0100 Subject: [PATCH 03/15] add yarn enable-storage-scan; add usedStorageBytes value to dataset compact --- app/controllers/DatasetController.scala | 3 +- .../WKRemoteDataStoreController.scala | 14 ++++---- app/models/dataset/Dataset.scala | 34 +++++++++++-------- app/models/dataset/DatasetService.scala | 3 +- conf/application.conf | 2 +- package.json | 1 + tools/postgres/dbtool.js | 13 +++++++ 7 files changed, 46 insertions(+), 24 deletions(-) diff --git a/app/controllers/DatasetController.scala b/app/controllers/DatasetController.scala index edb062c2086..42463293c8c 100755 --- a/app/controllers/DatasetController.scala +++ b/app/controllers/DatasetController.scala @@ -283,7 +283,8 @@ class DatasetController @Inject()(userService: UserService, searchQuery, request.identity.map(_._id), recursive.getOrElse(false), - limitOpt = limit + limitOpt = limit, + requestingUserOrga = request.identity.map(_._organization) ) } yield Json.toJson(datasetInfos) } else { diff --git a/app/controllers/WKRemoteDataStoreController.scala b/app/controllers/WKRemoteDataStoreController.scala index 9564ac496c2..dae9325e3bf 100644 --- a/app/controllers/WKRemoteDataStoreController.scala +++ b/app/controllers/WKRemoteDataStoreController.scala @@ -110,13 +110,13 @@ class WKRemoteDataStoreController @Inject()( teamIdsPerDataset <- Fox.combined(datasets.map(dataset => teamDAO.findAllowedTeamIdsForDataset(dataset.id))) unfinishedUploads = datasets.zip(teamIdsPerDataset).map { case (d, teamIds) => - new UnfinishedUpload("", - d.dataSourceId, - d.name, - d.folderId.toString, - d.created, - None, // Filled by datastore. - teamIds.map(_.toString)) + UnfinishedUpload("", + d.dataSourceId, + d.name, + d.folderId.toString, + d.created, + None, // Filled by datastore. + teamIds.map(_.toString)) } } yield Ok(Json.toJson(unfinishedUploads)) } diff --git a/app/models/dataset/Dataset.scala b/app/models/dataset/Dataset.scala index c0b8f56d60b..b60a5756ff5 100755 --- a/app/models/dataset/Dataset.scala +++ b/app/models/dataset/Dataset.scala @@ -231,18 +231,19 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA parsed <- parseAll(r) } yield parsed - def findAllCompactWithSearch(isActiveOpt: Option[Boolean] = None, - isUnreported: Option[Boolean] = None, - organizationIdOpt: Option[String] = None, - folderIdOpt: Option[ObjectId] = None, - uploaderIdOpt: Option[ObjectId] = None, - searchQuery: Option[String] = None, - requestingUserIdOpt: Option[ObjectId] = None, - includeSubfolders: Boolean = false, - statusOpt: Option[String] = None, - createdSinceOpt: Option[Instant] = None, - limitOpt: Option[Int] = None, - )(implicit ctx: DBAccessContext): Fox[List[DatasetCompactInfo]] = + def findAllCompactWithSearch( + isActiveOpt: Option[Boolean] = None, + isUnreported: Option[Boolean] = None, + organizationIdOpt: Option[String] = None, + folderIdOpt: Option[ObjectId] = None, + uploaderIdOpt: Option[ObjectId] = None, + searchQuery: Option[String] = None, + requestingUserIdOpt: Option[ObjectId] = None, + includeSubfolders: Boolean = false, + statusOpt: Option[String] = None, + createdSinceOpt: Option[Instant] = None, + limitOpt: Option[Int] = None, + requestingUserOrga: Option[String] = None)(implicit ctx: DBAccessContext): Fox[List[DatasetCompactInfo]] = for { selectionPredicates <- buildSelectionPredicates(isActiveOpt, isUnreported, @@ -290,7 +291,7 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA d.tags, cl.names AS colorLayerNames, sl.names AS segmentationLayerNames, - 0 AS usedStorageBytes -- // TODO + COALESCE(magStorage.storage, 0) + COALESCE(attachmentStorage.storage, 0) AS usedStorageBytes FROM (SELECT $columns FROM $existingCollectionName WHERE $selectionPredicates $limitQuery) d JOIN webknossos.organizations o @@ -303,6 +304,10 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA ON d._id = cl._dataset LEFT JOIN (SELECT _dataset, ARRAY_AGG(name ORDER BY name) AS names FROM webknossos.dataset_layers WHERE category = 'segmentation' GROUP BY _dataset) sl ON d._id = sl._dataset + LEFT JOIN (SELECT _dataset, COALESCE(SUM(usedStorageBytes), 0) AS storage FROM webknossos.organization_usedStorage_mags GROUP BY _dataset) magStorage + ON d._id = magStorage._dataset + LEFT JOIN (SELECT _dataset, COALESCE(SUM(usedStorageBytes), 0) AS storage FROM webknossos.organization_usedStorage_attachments GROUP BY _dataset) attachmentStorage + ON d._id = attachmentStorage._dataset """ rows <- run( query.as[ @@ -338,7 +343,8 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA isUnreported = DataSourceStatus.unreportedStatusList.contains(row._10), colorLayerNames = parseArrayLiteral(row._12), segmentationLayerNames = parseArrayLiteral(row._13), - usedStorageBytes = row._14, + // Only include usedStorage for datasets of your own organization. + usedStorageBytes = if (requestingUserOrga.contains(row._3)) row._14 else None, )) private def buildSelectionPredicates(isActiveOpt: Option[Boolean], diff --git a/app/models/dataset/DatasetService.scala b/app/models/dataset/DatasetService.scala index f028ea1119e..54588e4b90f 100644 --- a/app/models/dataset/DatasetService.scala +++ b/app/models/dataset/DatasetService.scala @@ -94,7 +94,8 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO, includeSubfolders = true, statusOpt = Some(DataSourceStatus.notYetUploaded), // Only list pending uploads since the two last weeks. - createdSinceOpt = Some(Instant.now - (14 days)) + createdSinceOpt = Some(Instant.now - (14 days)), + requestingUserOrga = Some(organizationId) ) ?~> "dataset.list.fetchFailed" def createAndSetUpDataset(datasetName: String, diff --git a/conf/application.conf b/conf/application.conf index b397c04350e..02bae200ae9 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -98,7 +98,7 @@ webKnossos { fetchUsedStorage { # Note that storage is only scanned for datastores that have `reportUsedStorageEnabled=TRUE` in postgres. rescanInterval = 24 hours # do not scan organizations whose last scan is more recent than this - tickerInterval = 10 minutes # scan some organizations at each tick + tickerInterval = 1 minutes # scan some organizations at each tick scansPerTick = 10 # scan x organizations at each tick } sampleOrganization { diff --git a/package.json b/package.json index a1c5670d965..01baf383ce4 100644 --- a/package.json +++ b/package.json @@ -104,6 +104,7 @@ "licenses-backend": "sbt dumpLicenseReport", "docs": "node_modules/.bin/documentation build --shallow frontend/javascripts/viewer/api/api_loader.ts frontend/javascripts/viewer/api/api_latest.ts --github --project-name \"WEBKNOSSOS Frontend API\" --format html --output public/docs/frontend-api", "refresh-schema": "./tools/postgres/dbtool.js refresh-schema && rm -f target/scala-2.13/src_managed/schema/com/scalableminds/webknossos/schema/Tables.scala", + "enable-storage-scan": "./tools/postgres/dbtool.js enable-storage-scan", "enable-jobs": "sed -i -e 's/jobsEnabled = false/jobsEnabled = true/g' ./conf/application.conf; sed -i -e 's/voxelyticsEnabled = false/voxelyticsEnabled = true/g' ./conf/application.conf; ./tools/postgres/dbtool.js enable-jobs", "disable-jobs": "sed -i -e 's/jobsEnabled = true/jobsEnabled = false/g' ./conf/application.conf; sed -i -e 's/voxelyticsEnabled = true/voxelyticsEnabled = false/g' ./conf/application.conf; ./tools/postgres/dbtool.js disable-jobs", "insert-local-datastore": "./tools/postgres/dbtool.js insert-local-datastore", diff --git a/tools/postgres/dbtool.js b/tools/postgres/dbtool.js index a8a125295b5..e57e3c2e453 100755 --- a/tools/postgres/dbtool.js +++ b/tools/postgres/dbtool.js @@ -420,6 +420,19 @@ program console.log("✨✨ Done"); }); +program + .command("enable-storage-scan") + .description("Activates dataset storage scan in WEBKNOSSOS for the default datastore.") + .action(() => { + console.log("Activating dataset storage scan in WEBKNOSSOS for the default datastore..."); + console.log( + callPsql( + `UPDATE webknossos.datastores SET reportUsedStorageEnabled = TRUE WHERE name = 'localhost'`, + ), + ); + console.log("✨✨ Done"); + }); + program .command("dump-schema ") .description("Dumps current schema into a folder") From c3f5549852c218498fc547fbd584de33ba059985 Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 29 Oct 2025 13:50:33 +0100 Subject: [PATCH 04/15] skip usedStorageBytes if it is zero --- app/models/dataset/Dataset.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/dataset/Dataset.scala b/app/models/dataset/Dataset.scala index b60a5756ff5..f4916ce5151 100755 --- a/app/models/dataset/Dataset.scala +++ b/app/models/dataset/Dataset.scala @@ -324,7 +324,7 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA String, String, String, - Option[Long])]) + Long)]) } yield rows.toList.map( row => @@ -344,7 +344,7 @@ class DatasetDAO @Inject()(sqlClient: SqlClient, datasetLayerDAO: DatasetLayerDA colorLayerNames = parseArrayLiteral(row._12), segmentationLayerNames = parseArrayLiteral(row._13), // Only include usedStorage for datasets of your own organization. - usedStorageBytes = if (requestingUserOrga.contains(row._3)) row._14 else None, + usedStorageBytes = if (requestingUserOrga.contains(row._3) && row._14 > 0) Some(row._14) else None, )) private def buildSelectionPredicates(isActiveOpt: Option[Boolean], From 054696b36c90f2703fc1f6bcba746e8e0831ec9b Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Wed, 29 Oct 2025 18:27:55 +0100 Subject: [PATCH 05/15] add tooltip and only show size for admins or ds managers --- .../advanced_dataset/dataset_table.tsx | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx index b47e6666205..d0048dca7ce 100644 --- a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx +++ b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx @@ -8,6 +8,7 @@ import type { TablePaginationConfig, } from "antd/lib/table/interface"; import classNames from "classnames"; +import FastTooltip from "components/fast_tooltip"; import FixedExpandableTable from "components/fixed_expandable_table"; import FormattedDate from "components/formatted_date"; import DatasetActionView, { @@ -288,9 +289,11 @@ class DatasetRenderer { } renderStorageColumn(): React.ReactNode { - return this.data.usedStorageBytes != null - ? formatCountToDataAmountUnit(this.data.usedStorageBytes, true) - : null; + return this.data.usedStorageBytes != null ? ( + + {formatCountToDataAmountUnit(this.data.usedStorageBytes, true)} + + ) : null; } renderTypeColumn(): React.ReactNode { return ; @@ -419,6 +422,7 @@ class DatasetTable extends React.PureComponent { // rendering). That's why it's not included in this.state (also it // would lead to infinite loops, too). currentPageData: RowRenderer[] = []; + isUserAdminOrDatasetManager: boolean = this.props.isUserAdmin || this.props.isUserDatasetManager; static getDerivedStateFromProps(nextProps: Props, prevState: State): Partial { const maybeSortedInfo: { sortedInfo: SorterResult } | EmptyObject = // Clear the sorting exactly when the search box is initially filled @@ -471,9 +475,7 @@ class DatasetTable extends React.PureComponent { }); const filterByHasLayers = (datasets: APIDatasetCompact[]) => - this.props.isUserAdmin || this.props.isUserDatasetManager - ? datasets - : datasets.filter((dataset) => dataset.isActive); + this.isUserAdminOrDatasetManager ? datasets : datasets.filter((dataset) => dataset.isActive); return filteredByTags(filterByMode(filterByHasLayers(this.props.datasets))); } @@ -613,6 +615,15 @@ class DatasetTable extends React.PureComponent { render: (_created, rowRenderer: RowRenderer) => rowRenderer.renderCreationDateColumn(), }, { + width: 200, + title: "Actions", + key: "actions", + fixed: "right", + render: (__, rowRenderer: RowRenderer) => rowRenderer.renderActionsColumn(), + }, + ]; + if (this.isUserAdminOrDatasetManager) { + const datasetStorageSizeColumn = { title: "Size", key: "size", width: 120, @@ -624,16 +635,10 @@ class DatasetTable extends React.PureComponent { ? rowRenderer.data.usedStorageBytes : 0, ), - sortOrder: sortedInfo.columnKey === "storage" ? sortedInfo.order : undefined, - }, - { - width: 200, - title: "Actions", - key: "actions", - fixed: "right", - render: (__, rowRenderer: RowRenderer) => rowRenderer.renderActionsColumn(), - }, - ]; + sortOrder: sortedInfo.columnKey === "size" ? sortedInfo.order : undefined, + }; + columns.splice(2, 0, datasetStorageSizeColumn); + } return ( From 3a5b987082d1c5f500379b6fadd62c4f1ff7e85b Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Wed, 29 Oct 2025 18:41:42 +0100 Subject: [PATCH 06/15] changelog --- unreleased_changes/9025.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 unreleased_changes/9025.md diff --git a/unreleased_changes/9025.md b/unreleased_changes/9025.md new file mode 100644 index 00000000000..f9d8849603c --- /dev/null +++ b/unreleased_changes/9025.md @@ -0,0 +1,2 @@ +### Added +- Display used storage for each dataset in the dashboard's dataset table. From 9e409139902b5f262f5dca93e47ab449e569ba27 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 30 Oct 2025 11:44:28 +0100 Subject: [PATCH 07/15] in yarn enable-storage-scan, also reset lastStorageScanTime for sample_organization --- tools/postgres/dbtool.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/postgres/dbtool.js b/tools/postgres/dbtool.js index e57e3c2e453..cfb7651afe7 100755 --- a/tools/postgres/dbtool.js +++ b/tools/postgres/dbtool.js @@ -430,6 +430,11 @@ program `UPDATE webknossos.datastores SET reportUsedStorageEnabled = TRUE WHERE name = 'localhost'`, ), ); + console.log( + callPsql( + `UPDATE webknossos.organizations SET lastStorageScanTime = '1970-01-01T00:00:00.000Z' WHERE _id = 'sample_organization'`, + ), + ); console.log("✨✨ Done"); }); From c80f5f355698d7871fd9db20008cc76046685551 Mon Sep 17 00:00:00 2001 From: Florian M Date: Thu, 30 Oct 2025 11:46:56 +0100 Subject: [PATCH 08/15] reset application.conf dev changes --- conf/application.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/application.conf b/conf/application.conf index 02bae200ae9..b397c04350e 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -98,7 +98,7 @@ webKnossos { fetchUsedStorage { # Note that storage is only scanned for datastores that have `reportUsedStorageEnabled=TRUE` in postgres. rescanInterval = 24 hours # do not scan organizations whose last scan is more recent than this - tickerInterval = 1 minutes # scan some organizations at each tick + tickerInterval = 10 minutes # scan some organizations at each tick scansPerTick = 10 # scan x organizations at each tick } sampleOrganization { From 09c4b014454cdc45643e61a43a7197239af28ccb Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Wed, 29 Oct 2025 23:25:44 +0100 Subject: [PATCH 09/15] check whether storage is reported in frontend --- .../advanced_dataset/dataset_table.tsx | 50 +++++++++---------- .../dataset/dataset_collection_context.tsx | 6 ++- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx index d0048dca7ce..1e19de2ede0 100644 --- a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx +++ b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx @@ -289,11 +289,11 @@ class DatasetRenderer { } renderStorageColumn(): React.ReactNode { - return this.data.usedStorageBytes != null ? ( - - {formatCountToDataAmountUnit(this.data.usedStorageBytes, true)} + return this.data.usedStorageBytes != null ? + ( + ? formatCountToDataAmountUnit(this.data.usedStorageBytes, true) - ) : null; + ) : null; } renderTypeColumn(): React.ReactNode { return ; @@ -429,11 +429,11 @@ class DatasetTable extends React.PureComponent { // (searchQuery changes from empty string to non-empty string) nextProps.searchQuery !== "" && prevState.prevSearchQuery === "" ? { - sortedInfo: { - columnKey: "", - order: "ascend", - }, - } + sortedInfo: { + columnKey: "", + order: "ascend", + }, + } : {}; return { prevSearchQuery: nextProps.searchQuery, @@ -566,20 +566,20 @@ class DatasetTable extends React.PureComponent { // and if the query is at least 3 characters long to avoid sorting *all* datasets isSearchQueryLongEnough && sortedInfo.columnKey == null ? _.chain([...filteredDataSource, ...activeSubfolders]) - .map((datasetOrFolder) => { - const diceCoefficient = dice(datasetOrFolder.name, this.props.searchQuery); - const rank = useLruRank ? datasetToRankMap.get(datasetOrFolder) || 0 : 0; - const rankCoefficient = 1 - rank / filteredDataSource.length; - const coefficient = (diceCoefficient + rankCoefficient) / 2; - return { - datasetOrFolder, - coefficient, - }; - }) - .sortBy("coefficient") - .map(({ datasetOrFolder }) => datasetOrFolder) - .reverse() - .value() + .map((datasetOrFolder) => { + const diceCoefficient = dice(datasetOrFolder.name, this.props.searchQuery); + const rank = useLruRank ? datasetToRankMap.get(datasetOrFolder) || 0 : 0; + const rankCoefficient = 1 - rank / filteredDataSource.length; + const coefficient = (diceCoefficient + rankCoefficient) / 2; + return { + datasetOrFolder, + coefficient, + }; + }) + .sortBy("coefficient") + .map(({ datasetOrFolder }) => datasetOrFolder) + .reverse() + .value() : dataSourceSortedByRank; const sortedDataSourceRenderers: RowRenderer[] = sortedDataSource.map((record) => isRecordADataset(record) @@ -622,7 +622,7 @@ class DatasetTable extends React.PureComponent { render: (__, rowRenderer: RowRenderer) => rowRenderer.renderActionsColumn(), }, ]; - if (this.isUserAdminOrDatasetManager) { + if (this.isUserAdminOrDatasetManager && context) { const datasetStorageSizeColumn = { title: "Size", key: "size", @@ -652,7 +652,7 @@ class DatasetTable extends React.PureComponent { contextMenuPosition={contextMenuPosition} datasetCollectionContext={context} editFolder={ - folderForContextMenu != null ? () => this.editFolder(folderForContextMenu) : () => {} + folderForContextMenu != null ? () => this.editFolder(folderForContextMenu) : () => { } } /> ; updateDatasetMutation: ReturnType; }; + usedStorageInOrga: number | undefined; }; export const DatasetCollectionContext = createContext( @@ -84,6 +85,7 @@ export default function DatasetCollectionContextProvider({ const [isChecking, setIsChecking] = useState(false); const isMutating = useIsMutating() > 0; const { data: folder, isError: didFolderLoadingError } = useFolderQuery(activeFolderId); + const usedStorageInOrga = useWkSelector((state) => state.activeOrganization?.usedStorageBytes); const [selectedDatasets, setSelectedDatasets] = useState([]); const [selectedFolder, setSelectedFolder] = useState(null); @@ -254,6 +256,7 @@ export default function DatasetCollectionContextProvider({ moveFolderMutation, updateDatasetMutation, }, + usedStorageInOrga, }), [ isChecking, @@ -279,6 +282,7 @@ export default function DatasetCollectionContextProvider({ getBreadcrumbs, selectedFolder, setGlobalSearchQuery, + usedStorageInOrga, ], ); From 642a404a5d9d2c2fe73402e587b457ce01f0c174 Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Thu, 30 Oct 2025 14:09:15 +0100 Subject: [PATCH 10/15] fix table entry --- .../advanced_dataset/dataset_table.tsx | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx index 1e19de2ede0..41a26dec0d7 100644 --- a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx +++ b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx @@ -289,11 +289,11 @@ class DatasetRenderer { } renderStorageColumn(): React.ReactNode { - return this.data.usedStorageBytes != null ? - ( - ? formatCountToDataAmountUnit(this.data.usedStorageBytes, true) + return this.data.usedStorageBytes != null ? ( + + {formatCountToDataAmountUnit(this.data.usedStorageBytes, true)} - ) : null; + ) : null; } renderTypeColumn(): React.ReactNode { return ; @@ -429,11 +429,11 @@ class DatasetTable extends React.PureComponent { // (searchQuery changes from empty string to non-empty string) nextProps.searchQuery !== "" && prevState.prevSearchQuery === "" ? { - sortedInfo: { - columnKey: "", - order: "ascend", - }, - } + sortedInfo: { + columnKey: "", + order: "ascend", + }, + } : {}; return { prevSearchQuery: nextProps.searchQuery, @@ -566,20 +566,20 @@ class DatasetTable extends React.PureComponent { // and if the query is at least 3 characters long to avoid sorting *all* datasets isSearchQueryLongEnough && sortedInfo.columnKey == null ? _.chain([...filteredDataSource, ...activeSubfolders]) - .map((datasetOrFolder) => { - const diceCoefficient = dice(datasetOrFolder.name, this.props.searchQuery); - const rank = useLruRank ? datasetToRankMap.get(datasetOrFolder) || 0 : 0; - const rankCoefficient = 1 - rank / filteredDataSource.length; - const coefficient = (diceCoefficient + rankCoefficient) / 2; - return { - datasetOrFolder, - coefficient, - }; - }) - .sortBy("coefficient") - .map(({ datasetOrFolder }) => datasetOrFolder) - .reverse() - .value() + .map((datasetOrFolder) => { + const diceCoefficient = dice(datasetOrFolder.name, this.props.searchQuery); + const rank = useLruRank ? datasetToRankMap.get(datasetOrFolder) || 0 : 0; + const rankCoefficient = 1 - rank / filteredDataSource.length; + const coefficient = (diceCoefficient + rankCoefficient) / 2; + return { + datasetOrFolder, + coefficient, + }; + }) + .sortBy("coefficient") + .map(({ datasetOrFolder }) => datasetOrFolder) + .reverse() + .value() : dataSourceSortedByRank; const sortedDataSourceRenderers: RowRenderer[] = sortedDataSource.map((record) => isRecordADataset(record) @@ -652,7 +652,7 @@ class DatasetTable extends React.PureComponent { contextMenuPosition={contextMenuPosition} datasetCollectionContext={context} editFolder={ - folderForContextMenu != null ? () => this.editFolder(folderForContextMenu) : () => { } + folderForContextMenu != null ? () => this.editFolder(folderForContextMenu) : () => {} } /> Date: Thu, 30 Oct 2025 14:15:20 +0100 Subject: [PATCH 11/15] rename column to match sidebar entry --- .../dashboard/advanced_dataset/dataset_table.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx index 41a26dec0d7..e01f131aae0 100644 --- a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx +++ b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx @@ -624,9 +624,9 @@ class DatasetTable extends React.PureComponent { ]; if (this.isUserAdminOrDatasetManager && context) { const datasetStorageSizeColumn = { - title: "Size", - key: "size", - width: 120, + title: "Used Storage", + key: "storage", + width: 150, render: (_: any, rowRenderer: RowRenderer) => { return isRecordADataset(rowRenderer.data) ? rowRenderer.renderStorageColumn() : null; }, @@ -635,7 +635,7 @@ class DatasetTable extends React.PureComponent { ? rowRenderer.data.usedStorageBytes : 0, ), - sortOrder: sortedInfo.columnKey === "size" ? sortedInfo.order : undefined, + sortOrder: sortedInfo.columnKey === "storage" ? sortedInfo.order : undefined, }; columns.splice(2, 0, datasetStorageSizeColumn); } From 8b654523bededc18d15727bf6e9e81f4b836ae4b Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Thu, 30 Oct 2025 14:25:37 +0100 Subject: [PATCH 12/15] check for used storage in orga --- .../dashboard/advanced_dataset/dataset_table.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx index e01f131aae0..b549b5fc38a 100644 --- a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx +++ b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx @@ -622,7 +622,11 @@ class DatasetTable extends React.PureComponent { render: (__, rowRenderer: RowRenderer) => rowRenderer.renderActionsColumn(), }, ]; - if (this.isUserAdminOrDatasetManager && context) { + if ( + this.isUserAdminOrDatasetManager && + context.usedStorageInOrga != null && + context.usedStorageInOrga > 0 + ) { const datasetStorageSizeColumn = { title: "Used Storage", key: "storage", From cb8d681f82fa9e072fcfcc05e0df963582d2d9ec Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Thu, 30 Oct 2025 14:46:50 +0100 Subject: [PATCH 13/15] update whether user is admin or DS manager --- .../dashboard/advanced_dataset/dataset_table.tsx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx index b549b5fc38a..a17c4379ca0 100644 --- a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx +++ b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx @@ -422,7 +422,9 @@ class DatasetTable extends React.PureComponent { // rendering). That's why it's not included in this.state (also it // would lead to infinite loops, too). currentPageData: RowRenderer[] = []; - isUserAdminOrDatasetManager: boolean = this.props.isUserAdmin || this.props.isUserDatasetManager; + getIsUserAdminOrDatasetManager(): boolean { + return this.props.isUserAdmin || this.props.isUserDatasetManager; + } static getDerivedStateFromProps(nextProps: Props, prevState: State): Partial { const maybeSortedInfo: { sortedInfo: SorterResult } | EmptyObject = // Clear the sorting exactly when the search box is initially filled @@ -475,7 +477,9 @@ class DatasetTable extends React.PureComponent { }); const filterByHasLayers = (datasets: APIDatasetCompact[]) => - this.isUserAdminOrDatasetManager ? datasets : datasets.filter((dataset) => dataset.isActive); + this.getIsUserAdminOrDatasetManager() + ? datasets + : datasets.filter((dataset) => dataset.isActive); return filteredByTags(filterByMode(filterByHasLayers(this.props.datasets))); } @@ -623,7 +627,7 @@ class DatasetTable extends React.PureComponent { }, ]; if ( - this.isUserAdminOrDatasetManager && + this.getIsUserAdminOrDatasetManager() && context.usedStorageInOrga != null && context.usedStorageInOrga > 0 ) { From f893f0bc671f24e741719371ea3fa3abe01e2fbd Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Thu, 30 Oct 2025 15:43:07 +0100 Subject: [PATCH 14/15] hopefully fix type checking tests --- frontend/javascripts/types/api_types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/javascripts/types/api_types.ts b/frontend/javascripts/types/api_types.ts index 856c28841d2..253c21e804b 100644 --- a/frontend/javascripts/types/api_types.ts +++ b/frontend/javascripts/types/api_types.ts @@ -266,13 +266,13 @@ export type APIDatasetCompactWithoutStatusAndLayerNames = Pick< | "lastUsedByUser" | "tags" | "isUnreported" - | "usedStorageBytes" >; export type APIDatasetCompact = APIDatasetCompactWithoutStatusAndLayerNames & { id: string; status: MutableAPIDataSourceBase["status"]; colorLayerNames: Array; segmentationLayerNames: Array; + usedStorageBytes?: number | null; }; export function convertDatasetToCompact(dataset: APIDataset): APIDatasetCompact { From df52face637af0464247df0bf8587bdbcf837c26 Mon Sep 17 00:00:00 2001 From: Charlie Meister Date: Thu, 30 Oct 2025 17:32:34 +0100 Subject: [PATCH 15/15] address UI feedback --- .../advanced_dataset/dataset_table.tsx | 34 ++++++++++++++++--- .../dashboard/folders/details_sidebar.tsx | 5 ++- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx index a17c4379ca0..8b9d2049600 100644 --- a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx +++ b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx @@ -1,6 +1,13 @@ -import { FileOutlined, FolderOpenOutlined, PlusOutlined, WarningOutlined } from "@ant-design/icons"; +import { + FileOutlined, + FolderOpenOutlined, + InfoCircleOutlined, + PlusOutlined, + WarningOutlined, +} from "@ant-design/icons"; import type { DatasetUpdater } from "admin/rest_api"; import { Dropdown, type MenuProps, Tag, Tooltip } from "antd"; +import { Space } from "antd/lib"; import type { ColumnType, FilterValue, @@ -290,7 +297,7 @@ class DatasetRenderer { renderStorageColumn(): React.ReactNode { return this.data.usedStorageBytes != null ? ( - + {formatCountToDataAmountUnit(this.data.usedStorageBytes, true)} ) : null; @@ -632,9 +639,28 @@ class DatasetTable extends React.PureComponent { context.usedStorageInOrga > 0 ) { const datasetStorageSizeColumn = { - title: "Used Storage", + title: ( + + Used Storage{" "} + + Storage used by this dataset within your organization. It may be zero because: +
    +
  • The storage hasn't been scanned yet.
    • +
  • The data is streamed from outside sources.
    • +
  • It’s counted in other datasets.
    • +
  • The dataset belongs to another organization.
    • +
+ + } + > + + {" "} + + ), key: "storage", - width: 150, + width: 200, render: (_: any, rowRenderer: RowRenderer) => { return isRecordADataset(rowRenderer.data) ? rowRenderer.renderStorageColumn() : null; }, diff --git a/frontend/javascripts/dashboard/folders/details_sidebar.tsx b/frontend/javascripts/dashboard/folders/details_sidebar.tsx index 31b445261be..5efa0fa32a9 100644 --- a/frontend/javascripts/dashboard/folders/details_sidebar.tsx +++ b/frontend/javascripts/dashboard/folders/details_sidebar.tsx @@ -204,7 +204,10 @@ function DatasetDetails({ selectedDataset }: { selectedDataset: APIDatasetCompac {fullDataset?.usedStorageBytes && fullDataset.usedStorageBytes > 10000 ? (
Used Storage
- +
{formatCountToDataAmountUnit(fullDataset.usedStorageBytes, true)}