diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 7ca096d8e14..c3e3f34b894 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -14203,11 +14203,16 @@ static IPropertyTreeIterator *deserializeFileAttrIterator(MemoryBuffer& mb, unsi if (options.includeField(DFUQResultField::size)) { - // JCSMORE - I am not sure what the point of this is, with or without it, a blank @size does not affect sort order - // and EclWatch seems to use the @size (DFUQResultField::origsize) for size column anyway - // See special handling in SerializeFileAttrOptions::readFields() to include origsize if size is requested + // Size field is notionally the size on disk const char *propName = getDFUQResultFieldName(DFUQResultField::size); - attr->setPropInt64(propName, attr->getPropInt64(getDFUQResultFieldName(DFUQResultField::origsize), -1));//Sort the files with empty size to front + attr->setPropInt64(propName, isCompressed(*attr) ? attr->getPropInt64(getDFUQResultFieldName(DFUQResultField::compressedsize), -1) : attr->getPropInt64(getDFUQResultFieldName(DFUQResultField::origsize), -1)); + } + + if (options.includeField(DFUQResultField::origsize)) + { + const char *propName = getDFUQResultFieldName(DFUQResultField::origsize); + if (!attr->hasProp(propName)) + attr->setPropInt64(propName, -1); } if (options.includeField(DFUQResultField::recordcount)) diff --git a/esp/services/ws_dfu/ws_dfuService.cpp b/esp/services/ws_dfu/ws_dfuService.cpp index 5f667e98c1a..463d7e3b9af 100644 --- a/esp/services/ws_dfu/ws_dfuService.cpp +++ b/esp/services/ws_dfu/ws_dfuService.cpp @@ -3769,6 +3769,7 @@ void CWsDfuEx::setDFUQuerySortOrder(IEspDFUQueryRequest& req, StringBuffer& sort static const std::unordered_map legacyMappings = { {"FileSize", "@DFUSFsize"}, + {"CompressedFileSize", "@compressedSize"}, {"ContentType", "@kind"}, {"IsCompressed", "@compressed"}, {"Records", "@recordcount"}, diff --git a/esp/src/eclwatch/DFUQueryWidget.js b/esp/src/eclwatch/DFUQueryWidget.js index d92cc2945da..7a7e5d2f717 100644 --- a/esp/src/eclwatch/DFUQueryWidget.js +++ b/esp/src/eclwatch/DFUQueryWidget.js @@ -737,7 +737,7 @@ define([ label: nlsHPCC.MinSkew, width: 60, formatter: function (value, row) { if (value) { - return "-" + Utility.formatDecimal(value / 100) + "%"; + return Utility.formatDecimal(value / 100, "-", "%"); } return ""; } @@ -746,7 +746,7 @@ define([ label: nlsHPCC.MaxSkew, width: 60, formatter: function (value, row) { if (value) { - return Utility.formatDecimal(value / 100) + "%"; + return Utility.formatDecimal(value / 100, "", "%"); } return ""; } diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 396559b626a..6a63cfff8b6 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -9,6 +9,7 @@ import { QuerySortItem } from "src/store/Store"; import nlsHPCC from "src/nlsHPCC"; import { useConfirm } from "../hooks/confirm"; import { useMyAccount } from "../hooks/user"; +import { formatCompression } from "../hooks/file"; import { HolyGrail } from "../layouts/HolyGrail"; import { pushParams } from "../util/history"; import { FluentPagedGrid, FluentPagedFooter, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; @@ -102,7 +103,6 @@ export const Files: React.FunctionComponent = ({ page = 1, store }) => { - const hasFilter = React.useMemo(() => Object.keys(filter).length > 0, [filter]); const [showFilter, setShowFilter] = React.useState(false); @@ -210,28 +210,27 @@ export const Files: React.FunctionComponent = ({ csvFormatter: (value, row) => row.IntRecordCount, }, FileSize: { - label: nlsHPCC.Size, + label: nlsHPCC.FileSize, formatter: (value, row) => { - return Utility.convertedSize(row.IntSize); + return Utility.convertedSize(row.IsCompressed ? row.CompressedFileSize : row.IntSize); }, - csvFormatter: (value, row) => row.IntSize, + csvFormatter: (value, row) => row.IsCompressed ? row.CompressedFileSize : row.IntSize, }, - CompressedFileSizeString: { - label: nlsHPCC.CompressedSize, + Compression: { + label: nlsHPCC.Compression, sortable: false, formatter: (value, row) => { - return Utility.convertedSize(row.CompressedFileSize); - }, - csvFormatter: (value, row) => row.CompressedFileSize, + return formatCompression(row); + } }, Parts: { label: nlsHPCC.Parts, width: 40, }, MinSkew: { - label: nlsHPCC.MinSkew, width: 60, formatter: (value, row) => value ? `-${Utility.formatDecimal(value / 100)}%` : "" + label: nlsHPCC.MinSkew, width: 60, formatter: (value, row) => value ? `${Utility.formatDecimal(value / 100, "-", "%")}` : "" }, MaxSkew: { - label: nlsHPCC.MaxSkew, width: 60, formatter: (value, row) => value ? `${Utility.formatDecimal(value / 100)}%` : "" + label: nlsHPCC.MaxSkew, width: 60, formatter: (value, row) => value ? `${Utility.formatDecimal(value / 100, "", "%")}` : "" }, Accessed: { label: uiState.isUTC ? nlsHPCC.LastAccessed : nlsHPCC.LastAccessedLocalTime, diff --git a/esp/src/src-react/components/IndexFileSummary.tsx b/esp/src/src-react/components/IndexFileSummary.tsx index 5f801c824ae..32184d5da2a 100644 --- a/esp/src/src-react/components/IndexFileSummary.tsx +++ b/esp/src/src-react/components/IndexFileSummary.tsx @@ -269,21 +269,21 @@ export const IndexFileSummary: React.FunctionComponent = label: nlsHPCC.File, originalSize: Utility.convertedSize(file?.FileSizeInt64), diskSize: Utility.convertedSize(file?.CompressedFileSize || file?.FileSizeInt64), - percentCompressed: ((file?.CompressedFileSize && file?.FileSizeInt64) ? Utility.formatDecimal(100 * file?.CompressedFileSize / file?.FileSizeInt64) : 0) + "%", + percentCompressed: ((file?.CompressedFileSize && file?.FileSizeInt64) ? Utility.formatDecimal(100 * file?.CompressedFileSize / file?.FileSizeInt64, "", "%") : ""), memorySize: (file?.ExtendedIndexInfo?.SizeMemoryBranches && file?.ExtendedIndexInfo?.SizeMemoryLeaves) ? Utility.convertedSize(file?.ExtendedIndexInfo?.SizeMemoryBranches + file?.ExtendedIndexInfo?.SizeMemoryLeaves) : "" }, { label: nlsHPCC.Branches, originalSize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeOriginalBranches) ?? "", diskSize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeDiskBranches) ?? "", - percentCompressed: file?.ExtendedIndexInfo?.BranchCompressionPercent ? Utility.formatDecimal(file.ExtendedIndexInfo.BranchCompressionPercent) + "%" : "", + percentCompressed: file?.ExtendedIndexInfo?.BranchCompressionPercent ? Utility.formatDecimal(file.ExtendedIndexInfo.BranchCompressionPercent, "", "%") : "", memorySize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeMemoryBranches) ?? "" }, { label: nlsHPCC.Data, originalSize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeOriginalData) ?? "", diskSize: (file?.ExtendedIndexInfo?.SizeDiskLeaves !== undefined && file?.ExtendedIndexInfo?.SizeDiskBlobs !== undefined) ? Utility.convertedSize(file?.ExtendedIndexInfo?.SizeDiskLeaves + file?.ExtendedIndexInfo?.SizeDiskBlobs) : "", - percentCompressed: file?.ExtendedIndexInfo?.DataCompressionPercent ? Utility.formatDecimal(file.ExtendedIndexInfo.DataCompressionPercent) + "%" : "", + percentCompressed: file?.ExtendedIndexInfo?.DataCompressionPercent ? Utility.formatDecimal(file.ExtendedIndexInfo.DataCompressionPercent, "", "%") : "", memorySize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeMemoryLeaves) ?? "" } ]} diff --git a/esp/src/src-react/components/LogicalFileSummary.tsx b/esp/src/src-react/components/LogicalFileSummary.tsx index 311ec9d6518..03aa8153f4c 100644 --- a/esp/src/src-react/components/LogicalFileSummary.tsx +++ b/esp/src/src-react/components/LogicalFileSummary.tsx @@ -7,7 +7,7 @@ import { formatCost } from "src/Session"; import * as Utility from "src/Utility"; import { getStateImageName, IFile } from "src/ESPLogicalFile"; import { useConfirm } from "../hooks/confirm"; -import { useFile } from "../hooks/file"; +import { formatCompression, useFile } from "../hooks/file"; import { useMyAccount } from "../hooks/user"; import { ShortVerticalDivider } from "./Common"; import { TableGroup } from "./forms/Groups"; @@ -218,11 +218,11 @@ export const LogicalFileSummary: React.FunctionComponent { const lIdx = sortByColumns.indexOf(l[0]); diff --git a/esp/src/src-react/hooks/file.ts b/esp/src/src-react/hooks/file.ts index 06548121c36..f69a10c77ab 100644 --- a/esp/src/src-react/hooks/file.ts +++ b/esp/src/src-react/hooks/file.ts @@ -1,11 +1,39 @@ import * as React from "react"; import { LogicalFile, WsDfu } from "@hpcc-js/comms"; import { scopedLogger } from "@hpcc-js/util"; +import * as Utility from "src/Utility"; import { singletonDebounce } from "../util/throttle"; import { useCounter } from "./util"; const logger = scopedLogger("../hooks/file.ts"); +export interface LogicalFileEx extends LogicalFile { + IntSize: number; +} + +function isLogicalFileEx(file: LogicalFile): file is LogicalFileEx { + return (file as LogicalFileEx).IntSize !== undefined; +} + +function formatRatio(isCompressed: boolean, compressedSize: number, totalSize: number): string { + if (!isCompressed || totalSize <= 0) return ""; + const ratio = compressedSize / totalSize; + if (!isFinite(ratio)) return ""; + return Utility.formatDecimal(100 - ratio * 100, "", "%"); +} + +function formatCompressionEx(file?: LogicalFileEx): string { + if (!file) return ""; + return formatRatio(file.IsCompressed, file.CompressedFileSize, file.IntSize); +} + +export function formatCompression(file?: LogicalFile): string { + if (!file) return ""; + if (file.isSuperfile) return ""; + if (isLogicalFileEx(file)) return formatCompressionEx(file); + return formatRatio(file.IsCompressed, file.CompressedFileSize, file.FileSizeInt64); +} + interface useFileResponse { file: LogicalFile; protectedBy: WsDfu.DFUFileProtect[]; diff --git a/esp/src/src/Utility.ts b/esp/src/src/Utility.ts index 5695fea6c44..32991e6e207 100644 --- a/esp/src/src/Utility.ts +++ b/esp/src/src/Utility.ts @@ -1191,10 +1191,10 @@ export function deleteCookie(name: string) { const d3FormatDecimal = d3Format(",.2f"); const d3FormatInt = d3Format(",.0f"); -export function formatDecimal(num: number): string { +export function formatDecimal(num: number, prefix: string = "", postfix: string = ""): string { if (!num) return ""; if (isNaN(num)) return num.toString(); - return d3FormatDecimal(num); + return prefix + d3FormatDecimal(num) + postfix; } export function formatNum(num: number): string { diff --git a/esp/src/tests/v9-files.spec.ts b/esp/src/tests/v9-files.spec.ts index d75aa7b633f..005da44c3d9 100644 --- a/esp/src/tests/v9-files.spec.ts +++ b/esp/src/tests/v9-files.spec.ts @@ -25,8 +25,8 @@ test.describe("V9 Files - Logical Files", () => { await expect(page.getByText("Description")).toBeVisible(); await expect(page.getByText("Cluster", { exact: true })).toBeVisible(); await expect(page.getByText("Records")).toBeVisible(); - await expect(page.getByText("Size", { exact: true })).toBeVisible(); - await expect(page.getByText("Compressed Size")).toBeVisible(); + await expect(page.getByText("File Size")).toBeVisible(); + await expect(page.getByText("Compression")).toBeVisible(); await expect(page.getByText("Parts")).toBeVisible(); await expect(page.getByText("Min Skew")).toBeVisible(); await expect(page.getByText("Max Skew")).toBeVisible(); diff --git a/testing/unittests/dalitests.cpp b/testing/unittests/dalitests.cpp index 353120a54eb..9679a163665 100644 --- a/testing/unittests/dalitests.cpp +++ b/testing/unittests/dalitests.cpp @@ -2683,6 +2683,9 @@ class DaliDFSIteratorTests : public CppUnit::TestFixture attr.setProp("@job", attrValue); attr.setProp("@owner", attrValue); attr.setProp("@workunit", attrValue); + attr.setPropBool("@rowCompressed", true); + attr.setPropInt64("@compressedSize", 9); + attr.setPropInt64("@size", 17); } } @@ -2857,6 +2860,9 @@ class DaliDFSIteratorTests : public CppUnit::TestFixture CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: Missing cost attributes", costAttrsPresent); bool dirAttrsPresent = attrs.hasProp("@directory"); CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: directory attribute should NOT be present", !dirAttrsPresent); + CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: size field missing", attrs.hasProp("@DFUSFsize")); + CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: compressed size field missing", attrs.hasProp("@compressedSize")); + CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: size should use compressed size", attrs.getPropInt64("@DFUSFsize", -1) == attrs.getPropInt64("@compressedSize", -1)); } } };