Merge branch 'hackathon/compare-perf' into tausbn/compare-perf-positive-negative-delta-iconongraphy

Author: tausbn

extensions/ql-vscode/src/log-insights/log-summary.ts

@@ -33,6 +33,7 @@ interface ResultEventBase extends SummaryEventBase {
 export interface ComputeSimple extends ResultEventBase {
   evaluationStrategy: "COMPUTE_SIMPLE";
   ra: Ra;
+  millis: number;
   pipelineRuns?: [PipelineRun];
   queryCausingWork?: string;
   dependencies: { [key: string]: string };

extensions/ql-vscode/src/log-insights/performance-comparison.ts

@@ -104,7 +104,6 @@ export class PerformanceOverviewScanner implements EvaluationLogScanner {
     switch (event.evaluationStrategy) {
       case "EXTENSIONAL":
       case "COMPUTED_EXTENSIONAL":
-      case "COMPUTE_SIMPLE":
       case "CACHACA": {
         // TODO: is CACHACA effectively the same as cache hit?
         break;
@@ -122,12 +121,17 @@ export class PerformanceOverviewScanner implements EvaluationLogScanner {
         break;
       }
       case "COMPUTE_RECURSIVE":
+      case "COMPUTE_SIMPLE":
       case "IN_LAYER": {
         const index = this.getPredicateIndex(event.predicateName);
         let totalTime = 0;
         let totalTuples = 0;
-        for (const millis of event.predicateIterationMillis ?? []) {
-          totalTime += millis;
+        if (event.evaluationStrategy === "COMPUTE_SIMPLE") {
+          totalTime += event.millis;
+        } else {
+          for (const millis of event.predicateIterationMillis ?? []) {
+            totalTime += millis;
+          }
         }
         const {
           timeCosts,

extensions/ql-vscode/src/view/compare-performance/ComparePerformance.tsx

@@ -28,6 +28,7 @@ interface OptionalValue {
 interface PredicateInfo extends OptionalValue {
   evaluationCount: number;
   iterationCount: number;
+  timeCost: number;
   pipelines: Record<string, PipelineSummary>;
 }
 
@@ -54,6 +55,7 @@ class ComparisonDataset {
         evaluationCount: 0,
         iterationCount: 0,
         tuples: 0,
+        timeCost: 0,
         absentReason: AbsentReason.NotSeen,
         pipelines: {},
       };
@@ -70,14 +72,15 @@ class ComparisonDataset {
     return {
       evaluationCount: data.evaluationCounts[index],
       iterationCount: data.iterationCounts[index],
+      timeCost: data.timeCosts[index],
       tuples: tupleCost,
       absentReason,
       pipelines: data.pipelineSummaryList[index],
     };
   }
 }
 
-function renderAbsoluteValue(x: OptionalValue) {
+function renderAbsoluteValue(x: PredicateInfo, metric: Metric) {
   switch (x.absentReason) {
     case AbsentReason.NotSeen:
       return <AbsentNumberCell>n/a</AbsentNumberCell>;
@@ -86,7 +89,12 @@ function renderAbsoluteValue(x: OptionalValue) {
     case AbsentReason.Sentinel:
       return <AbsentNumberCell>sentinel empty</AbsentNumberCell>;
     default:
-      return <NumberCell>{formatDecimal(x.tuples)}</NumberCell>;
+      return (
+        <NumberCell>
+          {formatDecimal(metric.get(x))}
+          {renderUnit(metric.unit)}
+        </NumberCell>
+      );
   }
 }
 
@@ -97,7 +105,8 @@ const DeltaPositive = styled(Codicon)`
 const DeltaNegative = styled(Codicon)`
   color: var(--vscode-problemsInfoIcon-foreground);
 `;
-function renderDelta(x: number) {
+
+function renderDelta(x: number, unit?: string) {
   const sign = x > 0 ? "+" : "";
   const symbol =
     x > 0 ? (
@@ -109,11 +118,16 @@ function renderDelta(x: number) {
     <NumberCell>
       {sign}
       {formatDecimal(x)}
+      {renderUnit(unit)}
       {symbol}
     </NumberCell>
   );
 }
 
+function renderUnit(unit: string | undefined) {
+  return unit == null ? "" : ` ${unit}`;
+}
+
 function orderBy<T>(fn: (x: T) => number | string) {
   return (x: T, y: T) => {
     const fx = fn(x);
@@ -199,7 +213,7 @@ const PipelineStepTR = styled.tr`
   }
 `;
 
-const SortOrderDropdown = styled.select``;
+const Dropdown = styled.select``;
 
 interface PipelineStepProps {
   before: number | undefined;
@@ -288,6 +302,37 @@ function getSortOrder(sortOrder: "delta" | "absDelta") {
   return orderBy((row: TRow) => row.diff);
 }
 
+interface Metric {
+  title: string;
+  get(info: PredicateInfo): number;
+  unit?: string;
+}
+
+const metrics: Record<string, Metric> = {
+  tuples: {
+    title: "Tuples in pipeline",
+    get: (info) => info.tuples,
+  },
+  time: {
+    title: "Time spent (milliseconds)",
+    get: (info) => info.timeCost,
+    unit: "ms",
+  },
+  evaluations: {
+    title: "Evaluations",
+    get: (info) => info.evaluationCount,
+  },
+  iterations: {
+    title: "Iterations (per evaluation)",
+    get: (info) =>
+      info.absentReason ? 0 : info.iterationCount / info.evaluationCount,
+  },
+  iterationsTotal: {
+    title: "Iterations (total)",
+    get: (info) => info.iterationCount,
+  },
+};
+
 function Chevron({ expanded }: { expanded: boolean }) {
   return <Codicon name={expanded ? "chevron-down" : "chevron-right"} />;
 }
@@ -333,6 +378,8 @@ export function ComparePerformance(_: Record<string, never>) {
 
   const [sortOrder, setSortOrder] = useState<"delta" | "absDelta">("delta");
 
+  const [metric, setMetric] = useState<Metric>(metrics.tuples);
+
   if (!datasets) {
     return <div>Loading performance comparison...</div>;
   }
@@ -350,7 +397,9 @@ export function ComparePerformance(_: Record<string, never>) {
     .map((name) => {
       const before = from.getTupleCountInfo(name);
       const after = to.getTupleCountInfo(name);
-      if (before.tuples === after.tuples) {
+      const beforeValue = metric.get(before);
+      const afterValue = metric.get(after);
+      if (beforeValue === afterValue) {
         return undefined!;
       }
       if (
@@ -362,7 +411,7 @@ export function ComparePerformance(_: Record<string, never>) {
           return undefined!;
         }
       }
-      const diff = after.tuples - before.tuples;
+      const diff = afterValue - beforeValue;
       return { name, before, after, diff };
     })
     .filter((x) => !!x)
@@ -373,8 +422,8 @@ export function ComparePerformance(_: Record<string, never>) {
   let totalDiff = 0;
 
   for (const row of rows) {
-    totalBefore += row.before.tuples;
-    totalAfter += row.after.tuples;
+    totalBefore += metric.get(row.before);
+    totalAfter += metric.get(row.after);
     totalDiff += row.diff;
   }
 
@@ -402,15 +451,28 @@ export function ComparePerformance(_: Record<string, never>) {
           </label>
         </WarningBox>
       )}
-      <SortOrderDropdown
+      Compare{" "}
+      <Dropdown
+        onChange={(e: ChangeEvent<HTMLSelectElement>) =>
+          setMetric(metrics[e.target.value])
+        }
+      >
+        {Object.entries(metrics).map(([key, value]) => (
+          <option key={key} value={key}>
+            {value.title}
+          </option>
+        ))}
+      </Dropdown>{" "}
+      sorted by{" "}
+      <Dropdown
         onChange={(e: ChangeEvent<HTMLSelectElement>) =>
           setSortOrder(e.target.value as "delta" | "absDelta")
         }
         value={sortOrder}
       >
         <option value="delta">Delta</option>
         <option value="absDelta">Absolute delta</option>
-      </SortOrderDropdown>
+      </Dropdown>
       <Table>
         <thead>
           <HeaderTR>
@@ -440,9 +502,9 @@ export function ComparePerformance(_: Record<string, never>) {
               <ChevronCell>
                 <Chevron expanded={expandedPredicates.has(row.name)} />
               </ChevronCell>
-              {renderAbsoluteValue(row.before)}
-              {renderAbsoluteValue(row.after)}
-              {renderDelta(row.diff)}
+              {renderAbsoluteValue(row.before, metric)}
+              {renderAbsoluteValue(row.after, metric)}
+              {renderDelta(row.diff, metric.unit)}
               <NameCell>{rowNames[rowIndex]}</NameCell>
             </PredicateTR>
             {expandedPredicates.has(row.name) && (