HPCC-35694 DFUQuery sort by compressed size

Signed-off-by: Gordon Smith<GordonJSmith@gmail.com>

Author: GordonSmith

.github/instructions/esp-service-interface.instructions.md

@@ -10,6 +10,11 @@ applyTo: '**/*.ecm'
 - Increment the `version` and `default_client_version` attributes of the `ESPservice` element.
 - Include the `exceptions_inline` attribute in any new `ESPresponse` and `ESPmethod` element.
 
+## Version Bumping
+- Always bump the `ESPservice` `version` and `default_client_version` when changing request/response schemas or behavior.
+- Use the next sequential version string (for example, if current is 1.67, bump to 1.68).
+- Keep changes backward compatible; do not remove or change existing fields without a version gate.
+
 ## Naming Conventions
 - Use `PascalCase` for element names.
 - Use lower case for native types such as `string`, `integer`, `boolean`, etc.

dali/base/dadfs.cpp

@@ -14146,6 +14146,14 @@ static IPropertyTreeIterator *deserializeFileAttrIterator(MemoryBuffer& mb, unsi
                 setIsCompressed(attr);
             }
 
+            if (options.includeField(DFUQResultField::compressedsize))
+            {
+                // Falls back to origsize if not compressed for more sensible sorting
+                // Really should distinguish between size, compressed size and disk size (where disk size === compressed size or orig size when uncompressed)
+                const char *compressedSizeProp = getDFUQResultFieldName(DFUQResultField::compressedsize);
+                attr->setPropInt64(compressedSizeProp, isCompressed(*attr) ? attr->getPropInt64(compressedSizeProp, -1) : attr->getPropInt64(getDFUQResultFieldName(DFUQResultField::origsize), -1));
+            }
+
             if (options.includeField(DFUQResultField::cost))
             {
                 // JCSMORE - cost could be computed in Dali instead - would simplify filtering/projecting code

esp/services/ws_dfu/ws_dfuService.cpp

@@ -2920,6 +2920,38 @@ __int64 CWsDfuEx::findPositionBySize(const __int64 size, bool descend, IArrayOf<
     return addToPos;
 }
 
+__int64 CWsDfuEx::findPositionByCompressedSize(const __int64 size, bool descend, IArrayOf<IEspDFULogicalFile>& LogicalFiles)
+{
+    __int64 addToPos = -1;
+
+    ForEachItemIn(i, LogicalFiles)
+    {
+        IEspDFULogicalFile& File = LogicalFiles.item(i);
+        __int64 nSize;
+        if (File.getIsCompressed())
+        {
+            nSize = File.getCompressedFileSize();
+        }
+        else
+        {
+            const char* sSize = File.getLongSize();
+            nSize = atoi64_l(sSize,strlen(sSize));
+        }
+        if (descend && size > nSize)
+        {
+            addToPos = i;
+            break;
+        }
+        if (!descend && size < nSize)
+        {
+            addToPos = i;
+            break;
+        }
+    }
+
+    return addToPos;
+}
+
 __int64 CWsDfuEx::findPositionByParts(const __int64 parts, bool descend, IArrayOf<IEspDFULogicalFile>& LogicalFiles)
 {
     __int64 addToPos = -1;
@@ -3333,6 +3365,7 @@ void CWsDfuEx::getAPageOfSortedLogicalFile(IEspContext &context, IUserDescriptor
             }
 
             __int64 recordSize=attr.getPropInt64("@recordSize",0), size=attr.getPropInt64("@size",-1);
+            __int64 compressedSize = attr.getPropInt64("@compressedSize", size);
 
             if (nFileSizeFrom > 0 && size < nFileSizeFrom)
                 continue;
@@ -3368,6 +3401,10 @@ void CWsDfuEx::getAPageOfSortedLogicalFile(IEspContext &context, IUserDescriptor
                 {
                     addToPos = findPositionBySize(size, descending, LogicalFileList);
                 }
+                else if (stricmp(sortBy, "CompressedFileSize")==0)
+                {
+                    addToPos = findPositionByCompressedSize(compressedSize, descending, LogicalFileList);
+                }
                 else if (stricmp(sortBy, "Parts")==0)
                 {
                     addToPos = findPositionByParts(parts, descending, LogicalFileList);
@@ -3769,6 +3806,7 @@ void CWsDfuEx::setDFUQuerySortOrder(IEspDFUQueryRequest& req, StringBuffer& sort
     static const std::unordered_map<std::string_view, std::string_view> legacyMappings =
     {
         {"FileSize", "@DFUSFsize"},
+        {"CompressedFileSize", "@compressedSize"},
         {"ContentType", "@kind"},
         {"IsCompressed", "@compressed"},
         {"Records", "@recordcount"},

esp/services/ws_dfu/ws_dfuService.hpp

@@ -215,6 +215,7 @@ class CWsDfuEx : public CWsDfu
                        IArrayOf<IEspDFULogicalFile>& LogicalFiles, int& addToPos, bool& reachLimit);
     __int64 findPositionByParts(const __int64 parts, bool decsend, IArrayOf<IEspDFULogicalFile>& LogicalFiles);
     __int64 findPositionBySize(const __int64 size, bool decsend, IArrayOf<IEspDFULogicalFile>& LogicalFiles);
+    __int64 findPositionByCompressedSize(const __int64 size, bool decsend, IArrayOf<IEspDFULogicalFile>& LogicalFiles);
     __int64 findPositionByRecords(const __int64 records, bool decsend, IArrayOf<IEspDFULogicalFile>& LogicalFiles);
     __int64 findPositionByName(const char *name, bool descend, IArrayOf<IEspDFULogicalFile>& LogicalFiles);
     __int64 findPositionByOwner(const char *owner, bool descend, IArrayOf<IEspDFULogicalFile>& LogicalFiles);

esp/src/src-react/components/Files.tsx

@@ -1,5 +1,6 @@
 import * as React from "react";
 import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Icon, Link } from "@fluentui/react";
+import { makeStyles, tokens } from "@fluentui/react-components";
 import { scopedLogger } from "@hpcc-js/util";
 import * as WsDfu from "src/WsDfu";
 import { CreateDFUQueryStore, Get } from "src/ESPLogicalFile";
@@ -24,6 +25,13 @@ import { SizeMe } from "../layouts/SizeMe";
 
 const logger = scopedLogger("src-react/components/Files.tsx");
 
+const useStyles = makeStyles({
+    dimmed: {
+        opacity: "0.666",
+        color: tokens.colorNeutralForeground3
+    }
+});
+
 const FilterFields: Fields = {
     "LogicalName": { type: "string", label: nlsHPCC.Name, placeholder: nlsHPCC.somefile },
     "Description": { type: "string", label: nlsHPCC.Description, placeholder: nlsHPCC.SomeDescription },
@@ -103,6 +111,7 @@ export const Files: React.FunctionComponent<FilesProps> = ({
     store
 }) => {
 
+    const styles = useStyles();
     const hasFilter = React.useMemo(() => Object.keys(filter).length > 0, [filter]);
 
     const [showFilter, setShowFilter] = React.useState(false);
@@ -216,10 +225,23 @@ export const Files: React.FunctionComponent<FilesProps> = ({
                 },
                 csvFormatter: (value, row) => row.IntSize,
             },
-            CompressedFileSizeString: {
+            CompressedFileSize: {
                 label: nlsHPCC.CompressedSize,
                 formatter: (value, row) => {
-                    return Utility.convertedSize(row.CompressedFileSize);
+                    if (row.IsCompressed) {
+                        return Utility.convertedSize(row.CompressedFileSize);
+                    }
+                    const fileSize = Utility.convertedSize(row.IntSize);
+                    if (fileSize) {
+                        return `[${fileSize}]`;
+                    }
+                    return "";
+                },
+                className: (value: any, row: any) => {
+                    if (!row.IsCompressed) {
+                        return styles.dimmed;
+                    }
+                    return "";
                 },
                 csvFormatter: (value, row) => row.CompressedFileSize,
             },