HPCC-35694 DFUQuery sort by compressed size

.github/instructions/esp-service-interface.instructions.md

@@ -11,6 +11,11 @@ applyTo: '**/*.ecm'
 - Include the `exceptions_inline` attribute in any new `ESPresponse` element.
 - Include the `exceptions_inline` attribute in any new `ESPmethod` element only if its parent `ESPservice` element doesn't have an `exceptions_inline` attribute.
 
+## Version Bumping
+- Always bump the `ESPservice` `version` and `default_client_version` when changing request/response schemas or behavior.
+- Use the next sequential version string (for example, if current is 1.67, bump to 1.68).
+- Keep changes backward compatible; do not remove or change existing fields without a version gate.
+
 ## Naming Conventions
 - Use `PascalCase` for element names.
 - Use lower case for native types such as `string`, `integer`, `boolean`, etc.

dali/base/dadfs.cpp

@@ -14203,11 +14203,15 @@ static IPropertyTreeIterator *deserializeFileAttrIterator(MemoryBuffer& mb, unsi
 
             if (options.includeField(DFUQResultField::size))
             {
-                // JCSMORE - I am not sure what the point of this is, with or without it, a blank @size does not affect sort order
-                // and EclWatch seems to use the @size (DFUQResultField::origsize) for size column anyway
-                // See special handling in SerializeFileAttrOptions::readFields() to include origsize if size is requested
+                // Size field is notionally the size on disk
                 const char *propName = getDFUQResultFieldName(DFUQResultField::size);
-                attr->setPropInt64(propName, attr->getPropInt64(getDFUQResultFieldName(DFUQResultField::origsize), -1));//Sort the files with empty size to front
+                attr->setPropInt64(propName, isCompressed(*attr) ? attr->getPropInt64(getDFUQResultFieldName(DFUQResultField::compressedsize), -1) : attr->getPropInt64(getDFUQResultFieldName(DFUQResultField::origsize), -1));
+            }
+
+            if (options.includeField(DFUQResultField::origsize))
+            {
+                const char *propName = getDFUQResultFieldName(DFUQResultField::origsize);
+                attr->setPropInt64(propName, attr->getPropInt64(getDFUQResultFieldName(DFUQResultField::origsize), -1));
             }
 
             if (options.includeField(DFUQResultField::recordcount))

esp/services/ws_dfu/ws_dfuService.cpp

@@ -3769,6 +3769,8 @@ void CWsDfuEx::setDFUQuerySortOrder(IEspDFUQueryRequest& req, StringBuffer& sort
     static const std::unordered_map<std::string_view, std::string_view> legacyMappings =
     {
         {"FileSize", "@DFUSFsize"},
+        {"Size", "@origsize"},
+        {"CompressedFileSize", "@compressedSize"},
         {"ContentType", "@kind"},
         {"IsCompressed", "@compressed"},
         {"Records", "@recordcount"},

esp/src/eclwatch/DFUQueryWidget.js

@@ -737,7 +737,7 @@ define([
                         label: nlsHPCC.MinSkew, width: 60,
                         formatter: function (value, row) {
                             if (value) {
-                                return "-" + Utility.formatDecimal(value / 100) + "%";
+                                return Utility.formatDecimal(value / 100, "-", "%");
                             }
                             return "";
                         }
@@ -746,7 +746,7 @@ define([
                         label: nlsHPCC.MaxSkew, width: 60,
                         formatter: function (value, row) {
                             if (value) {
-                                return Utility.formatDecimal(value / 100) + "%";
+                                return Utility.formatDecimal(value / 100, "", "%");
                             }
                             return "";
                         }

esp/src/src-react/components/Files.tsx

@@ -9,6 +9,7 @@ import { QuerySortItem } from "src/store/Store";
 import nlsHPCC from "src/nlsHPCC";
 import { useConfirm } from "../hooks/confirm";
 import { useMyAccount } from "../hooks/user";
+import { formatCompression } from "../hooks/file";
 import { HolyGrail } from "../layouts/HolyGrail";
 import { pushParams } from "../util/history";
 import { FluentPagedGrid, FluentPagedFooter, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid";
@@ -102,7 +103,6 @@ export const Files: React.FunctionComponent<FilesProps> = ({
     page = 1,
     store
 }) => {
-
     const hasFilter = React.useMemo(() => Object.keys(filter).length > 0, [filter]);
 
     const [showFilter, setShowFilter] = React.useState(false);
@@ -210,28 +210,27 @@ export const Files: React.FunctionComponent<FilesProps> = ({
                 csvFormatter: (value, row) => row.IntRecordCount,
             },
             FileSize: {
-                label: nlsHPCC.Size,
+                label: nlsHPCC.FileSize,
                 formatter: (value, row) => {
-                    return Utility.convertedSize(row.IntSize);
+                    return Utility.convertedSize(row.IsCompressed ? row.CompressedFileSize : row.IntSize);
                 },
-                csvFormatter: (value, row) => row.IntSize,
+                csvFormatter: (value, row) => row.IsCompressed ? row.CompressedFileSize : row.IntSize,
             },
-            CompressedFileSizeString: {
-                label: nlsHPCC.CompressedSize,
+            Compression: {
+                label: nlsHPCC.Compression,
                 sortable: false,
                 formatter: (value, row) => {
-                    return Utility.convertedSize(row.CompressedFileSize);
-                },
-                csvFormatter: (value, row) => row.CompressedFileSize,
+                    return formatCompression(row);
+                }
             },
             Parts: {
                 label: nlsHPCC.Parts, width: 40,
             },
             MinSkew: {
-                label: nlsHPCC.MinSkew, width: 60, formatter: (value, row) => value ? `-${Utility.formatDecimal(value / 100)}%` : ""
+                label: nlsHPCC.MinSkew, width: 60, formatter: (value, row) => value ? `${Utility.formatDecimal(value / 100, "-", "%")}` : ""
             },
             MaxSkew: {
-                label: nlsHPCC.MaxSkew, width: 60, formatter: (value, row) => value ? `${Utility.formatDecimal(value / 100)}%` : ""
+                label: nlsHPCC.MaxSkew, width: 60, formatter: (value, row) => value ? `${Utility.formatDecimal(value / 100, "", "%")}` : ""
             },
             Accessed: {
                 label: uiState.isUTC ? nlsHPCC.LastAccessed : nlsHPCC.LastAccessedLocalTime,

esp/src/src-react/components/IndexFileSummary.tsx

@@ -269,21 +269,21 @@ export const IndexFileSummary: React.FunctionComponent<IndexFileSummaryProps> =
                         label: nlsHPCC.File,
                         originalSize: Utility.convertedSize(file?.FileSizeInt64),
                         diskSize: Utility.convertedSize(file?.CompressedFileSize || file?.FileSizeInt64),
-                        percentCompressed: ((file?.CompressedFileSize && file?.FileSizeInt64) ? Utility.formatDecimal(100 * file?.CompressedFileSize / file?.FileSizeInt64) : 0) + "%",
+                        percentCompressed: ((file?.CompressedFileSize && file?.FileSizeInt64) ? Utility.formatDecimal(100 * file?.CompressedFileSize / file?.FileSizeInt64, "", "%") : ""),
                         memorySize: (file?.ExtendedIndexInfo?.SizeMemoryBranches && file?.ExtendedIndexInfo?.SizeMemoryLeaves) ? Utility.convertedSize(file?.ExtendedIndexInfo?.SizeMemoryBranches + file?.ExtendedIndexInfo?.SizeMemoryLeaves) : ""
                     },
                     {
                         label: nlsHPCC.Branches,
                         originalSize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeOriginalBranches) ?? "",
                         diskSize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeDiskBranches) ?? "",
-                        percentCompressed: file?.ExtendedIndexInfo?.BranchCompressionPercent ? Utility.formatDecimal(file.ExtendedIndexInfo.BranchCompressionPercent) + "%" : "",
+                        percentCompressed: file?.ExtendedIndexInfo?.BranchCompressionPercent ? Utility.formatDecimal(file.ExtendedIndexInfo.BranchCompressionPercent, "", "%") : "",
                         memorySize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeMemoryBranches) ?? ""
                     },
                     {
                         label: nlsHPCC.Data,
                         originalSize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeOriginalData) ?? "",
                         diskSize: (file?.ExtendedIndexInfo?.SizeDiskLeaves !== undefined && file?.ExtendedIndexInfo?.SizeDiskBlobs !== undefined) ? Utility.convertedSize(file?.ExtendedIndexInfo?.SizeDiskLeaves + file?.ExtendedIndexInfo?.SizeDiskBlobs) : "",
-                        percentCompressed: file?.ExtendedIndexInfo?.DataCompressionPercent ? Utility.formatDecimal(file.ExtendedIndexInfo.DataCompressionPercent) + "%" : "",
+                        percentCompressed: file?.ExtendedIndexInfo?.DataCompressionPercent ? Utility.formatDecimal(file.ExtendedIndexInfo.DataCompressionPercent, "", "%") : "",
                         memorySize: Utility.convertedSize(file?.ExtendedIndexInfo?.SizeMemoryLeaves) ?? ""
                     }
                 ]}

esp/src/src-react/components/LogicalFileSummary.tsx

@@ -7,7 +7,7 @@ import { formatCost } from "src/Session";
 import * as Utility from "src/Utility";
 import { getStateImageName, IFile } from "src/ESPLogicalFile";
 import { useConfirm } from "../hooks/confirm";
-import { useFile } from "../hooks/file";
+import { formatCompression, useFile } from "../hooks/file";
 import { useMyAccount } from "../hooks/user";
 import { ShortVerticalDivider } from "./Common";
 import { TableGroup } from "./forms/Groups";
@@ -218,11 +218,11 @@ export const LogicalFileSummary: React.FunctionComponent<LogicalFileSummaryProps
                 "ContentType": { label: nlsHPCC.ContentType, type: "string", value: file?.ContentType, readonly: true },
                 "KeyType": { label: nlsHPCC.KeyType, type: "string", value: file?.KeyType, readonly: true },
                 "Format": { label: nlsHPCC.Format, type: "string", value: file?.Format, readonly: true },
+                "Filesize": { label: nlsHPCC.FileSize, type: "string", value: file?.Filesize, readonly: true },
                 "IsCompressed": { label: nlsHPCC.IsCompressed, type: "checkbox", value: file?.IsCompressed, readonly: true },
                 "CompressedFileSizeString": { label: nlsHPCC.CompressedFileSize, type: "string", value: file?.CompressedFileSize ? Utility.safeFormatNum(file?.CompressedFileSize) : "", readonly: true },
                 "CompressionType": { label: nlsHPCC.CompressionType, type: "string", value: file?.CompressionType, readonly: true },
-                "Filesize": { label: nlsHPCC.FileSize, type: "string", value: file?.Filesize, readonly: true },
-                "PercentCompressed": { label: nlsHPCC.PercentCompressed, type: "string", value: file?.PercentCompressed, readonly: true },
+                "Compression": { label: nlsHPCC.Compression, type: "string", value: formatCompression(file), readonly: true },
                 "Modified": { label: nlsHPCC.Modified, type: "string", value: file?.Modified, readonly: true },
                 "ExpirationDate": { label: nlsHPCC.ExpirationDate, type: "string", value: file?.ExpirationDate, readonly: true },
                 "ExpireDays": { label: nlsHPCC.ExpireDays, type: "string", value: file?.ExpireDays ? file?.ExpireDays.toString() : "", readonly: true },

esp/src/src-react/components/MetricsPropertiesTables.tsx

@@ -164,7 +164,7 @@ export const MetricsPropertiesTables: React.FunctionComponent<MetricsPropertiesT
                     default:
                         rowValue = row.Value;
                 }
-                scopeProps.push([row.Key, rowValue, row.Avg, row.Min, row.Max, row.Delta, row.StdDev === undefined ? "" : `${row.StdDev} (${formatDecimal(row.StdDevs)}σ)`, row.SkewMin, row.SkewMax, row.NodeMin, row.NodeMax, row.StdDevs]);
+                scopeProps.push([row.Key, rowValue, row.Avg, row.Min, row.Max, row.Delta, row.StdDev === undefined ? "" : `${row.StdDev} ${formatDecimal(row.StdDevs, "(", "σ)")}`, row.SkewMin, row.SkewMax, row.NodeMin, row.NodeMax, row.StdDevs]);
             }
             scopeProps.sort((l, r) => {
                 const lIdx = sortByColumns.indexOf(l[0]);

esp/src/src-react/hooks/file.ts

@@ -1,11 +1,39 @@
 import * as React from "react";
 import { LogicalFile, WsDfu } from "@hpcc-js/comms";
 import { scopedLogger } from "@hpcc-js/util";
+import * as Utility from "src/Utility";
 import { singletonDebounce } from "../util/throttle";
 import { useCounter } from "./util";
 
 const logger = scopedLogger("../hooks/file.ts");
 
+export interface LogicalFileEx extends LogicalFile {
+    IntSize: number;
+}
+
+function isLogicalFileEx(file: LogicalFile): file is LogicalFileEx {
+    return (file as LogicalFileEx).IntSize !== undefined;
+}
+
+function formatRatio(isCompressed: boolean, compressedSize: number, totalSize: number): string {
+    if (!isCompressed || totalSize <= 0) return "";
+    const ratio = compressedSize / totalSize;
+    if (!isFinite(ratio)) return "";
+    return Utility.formatDecimal(100 - ratio * 100, "", "%");
+}
+
+function formatCompressionEx(file?: LogicalFileEx): string {
+    if (!file) return "";
+    return formatRatio(file.IsCompressed, file.CompressedFileSize, file.IntSize);
+}
+
+export function formatCompression(file?: LogicalFile): string {
+    if (!file) return "";
+    if (file.isSuperfile) return "";
+    if (isLogicalFileEx(file)) return formatCompressionEx(file);
+    return formatRatio(file.IsCompressed, file.CompressedFileSize, file.FileSizeInt64);
+}
+
 interface useFileResponse {
     file: LogicalFile;
     protectedBy: WsDfu.DFUFileProtect[];

esp/src/src/Utility.ts

@@ -1191,10 +1191,10 @@ export function deleteCookie(name: string) {
 const d3FormatDecimal = d3Format(",.2f");
 const d3FormatInt = d3Format(",.0f");
 
-export function formatDecimal(num: number): string {
+export function formatDecimal(num: number, prefix: string = "", postfix: string = ""): string {
     if (!num) return "";
     if (isNaN(num)) return num.toString();
-    return d3FormatDecimal(num);
+    return prefix + d3FormatDecimal(num) + postfix;
 }
 
 export function formatNum(num: number): string {

esp/src/tests/v9-files.spec.ts

@@ -25,8 +25,8 @@ test.describe("V9 Files - Logical Files", () => {
         await expect(page.getByText("Description")).toBeVisible();
         await expect(page.getByText("Cluster", { exact: true })).toBeVisible();
         await expect(page.getByText("Records")).toBeVisible();
-        await expect(page.getByText("Size", { exact: true })).toBeVisible();
-        await expect(page.getByText("Compressed Size")).toBeVisible();
+        await expect(page.getByText("File Size")).toBeVisible();
+        await expect(page.getByText("Compression")).toBeVisible();
         await expect(page.getByText("Parts")).toBeVisible();
         await expect(page.getByText("Min Skew")).toBeVisible();
         await expect(page.getByText("Max Skew")).toBeVisible();

testing/unittests/dalitests.cpp

@@ -2683,6 +2683,9 @@ class DaliDFSIteratorTests : public CppUnit::TestFixture
             attr.setProp("@job", attrValue);
             attr.setProp("@owner", attrValue);
             attr.setProp("@workunit", attrValue);
+            attr.setPropBool("@rowCompressed", true);
+            attr.setPropInt64("@compressedSize", 9);
+            attr.setPropInt64("@size", 17);
         }
     }
 
@@ -2857,6 +2860,9 @@ class DaliDFSIteratorTests : public CppUnit::TestFixture
             CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: Missing cost attributes", costAttrsPresent);
             bool dirAttrsPresent = attrs.hasProp("@directory");
             CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: directory attribute should NOT be present", !dirAttrsPresent);
+            CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: size field missing", attrs.hasProp("@DFUSFsize"));
+            CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: compressed size field missing", attrs.hasProp("@compressedSize"));
+            CPPUNIT_ASSERT_MESSAGE("testGetLogicalFilesSorted: size should use compressed size", attrs.getPropInt64("@DFUSFsize", -1) == attrs.getPropInt64("@compressedSize", -1));
         }
     }
 };