Stats info (#10)

* add first and last occurrence info to stats

* fix search case-sensitivity

Author: vish9812

Taskfile.yml

@@ -19,9 +19,14 @@ tasks:
       - cmd:install
 
   test:
-      desc: run tests
-      deps:
-        - ui:test
+    desc: run tests
+    deps:
+      - ui:test
+
+  run:
+    desc: run the UI
+    deps:
+      - ui:run
 
   build:
     desc: transpile, bundle and create the release zip

cmd/src/commands/summary/index.ts

@@ -3,21 +3,36 @@ import { Table } from "console-table-printer";
 import { parseArgs } from "util";
 import { cpus } from "node:os";
 import type { ICmd } from "@al/cmd/utils/cmd-runner";
-import { type ITask, type IResult } from "./worker";
+import type {
+  ITask,
+  IResult,
+  IGroupedMsg,
+  ISummaryMap,
+  ISummaryMapGeneric,
+} from "./worker";
 import WorkerPool from "@al/cmd/utils/worker-pool";
 import fileHelper from "@al/cmd/utils/file-helper";
-import type {
-  GroupedMsg,
-  Summary as SummaryData,
-  SummaryMap,
-} from "@al/ui/models/logData";
-import LogData from "@al/ui/models/logData";
 
 let workerURL = new URL("worker.ts", import.meta.url);
 
-interface IStats extends Omit<IResult, "filePath"> {
+interface IStats extends Omit<IResult, "filePath" | "dataMap"> {
   minTimeFile: string;
   maxTimeFile: string;
+  dataMap: IStatsSummaryMap;
+}
+
+interface IStatsGroupedMsg extends IGroupedMsg {
+  firstFile: string;
+  lastFile: string;
+}
+
+type IStatsSummaryMap = ISummaryMapGeneric<IStatsGroupedMsg>;
+
+interface ISummary {
+  msgs: IStatsGroupedMsg[];
+  httpCodes: IStatsGroupedMsg[];
+  jobs: IStatsGroupedMsg[];
+  plugins: IStatsGroupedMsg[];
 }
 
 const stats: IStats = {
@@ -27,10 +42,10 @@ const stats: IStats = {
   minTimeFile: "",
   size: 0,
   dataMap: {
-    httpCodes: new Map<string, GroupedMsg>(),
-    jobs: new Map<string, GroupedMsg>(),
-    msgs: new Map<string, GroupedMsg>(),
-    plugins: new Map<string, GroupedMsg>(),
+    httpCodes: new Map<string, IStatsGroupedMsg>(),
+    jobs: new Map<string, IStatsGroupedMsg>(),
+    msgs: new Map<string, IStatsGroupedMsg>(),
+    plugins: new Map<string, IStatsGroupedMsg>(),
   },
 };
 
@@ -109,7 +124,7 @@ async function processLogs() {
   await readFiles(filePaths);
   console.log("=========End Read Files=========");
 
-  const summary = LogData.initSummary(stats.dataMap);
+  const summary = initSummary(stats.dataMap);
 
   writeContent(summary);
 }
@@ -155,37 +170,64 @@ function processFileResponse(fileStats: IResult) {
 
   stats.size += fileStats.size;
 
-  initSummaryMap(fileStats.dataMap);
+  initSummaryMap(fileStats.dataMap, fileStats.filePath);
 }
 
-function initSummaryMap(dataMap: SummaryMap) {
-  mergeIntoOverallMap(dataMap.httpCodes, stats.dataMap.httpCodes);
-  mergeIntoOverallMap(dataMap.jobs, stats.dataMap.jobs);
-  mergeIntoOverallMap(dataMap.msgs, stats.dataMap.msgs);
-  mergeIntoOverallMap(dataMap.plugins, stats.dataMap.plugins);
+function initSummaryMap(dataMap: ISummaryMap, filePath: string) {
+  mergeIntoOverallMap(dataMap.httpCodes, stats.dataMap.httpCodes, filePath);
+  mergeIntoOverallMap(dataMap.jobs, stats.dataMap.jobs, filePath);
+  mergeIntoOverallMap(dataMap.msgs, stats.dataMap.msgs, filePath);
+  mergeIntoOverallMap(dataMap.plugins, stats.dataMap.plugins, filePath);
 }
 
 function mergeIntoOverallMap(
-  fileMap: Map<string, GroupedMsg>,
-  overallMap: Map<string, GroupedMsg>
+  fileMap: Map<string, IGroupedMsg>,
+  overallMap: Map<string, IStatsGroupedMsg>,
+  filePath: string
 ) {
   for (const [k, v] of fileMap) {
     if (!overallMap.has(k)) {
       overallMap.set(k, {
         msg: v.msg,
         hasErrors: false,
-        logs: [],
         logsCount: 0,
+        firstTime: v.firstTime,
+        lastTime: v.lastTime,
+        firstFile: filePath,
+        lastFile: filePath,
       });
     }
 
     const grpOverall = overallMap.get(k)!;
-    grpOverall.hasErrors = grpOverall.hasErrors || v.hasErrors;
+    grpOverall.hasErrors ||= v.hasErrors;
     grpOverall.logsCount += v.logsCount!;
+
+    if (v.firstTime < grpOverall.firstTime) {
+      grpOverall.firstTime = v.firstTime;
+      grpOverall.firstFile = filePath;
+    }
+
+    if (v.lastTime > grpOverall.lastTime) {
+      grpOverall.lastTime = v.lastTime;
+      grpOverall.lastFile = filePath;
+    }
   }
 }
 
-function writeContent(summary: SummaryData) {
+function summarySorterFn(a: IStatsGroupedMsg, b: IStatsGroupedMsg) {
+  return b.logsCount - a.logsCount;
+}
+
+function initSummary(summaryMap: IStatsSummaryMap): ISummary {
+  return {
+    msgs: [...summaryMap.msgs.values()].sort(summarySorterFn),
+    httpCodes: [...summaryMap.httpCodes.values()].sort(summarySorterFn),
+    jobs: [...summaryMap.jobs.values()].sort(summarySorterFn),
+    plugins: [...summaryMap.plugins.values()].sort(summarySorterFn),
+  };
+}
+
+function writeContent(summary: ISummary) {
   console.log();
 
   stats.size = prettyBytes(stats.size) as any;
@@ -217,15 +259,15 @@ function writeContent(summary: SummaryData) {
   writeGroupedMsgs(summary.plugins, "Plugins");
 }
 
-function writeGroupedMsgs(grpMsgs: GroupedMsg[], title: string) {
+function writeGroupedMsgs(grpMsgs: IStatsGroupedMsg[], title: string) {
   console.log();
 
   const table = new Table({
     columns: [
       { name: "msg", title: title, alignment: "left" },
       { name: "logsCount", title: "Count" },
     ],
-    disabledColumns: ["hasErrors", "logs"],
+    disabledColumns: ["hasErrors"],
   });
 
   for (const grp of grpMsgs.slice(0, flags.topLogsCount)) {

cmd/src/commands/summary/worker.ts

@@ -1,10 +1,24 @@
 import { parentPort } from "node:worker_threads";
-import LogData, {
-  type GroupedMsg,
-  type SummaryMap,
-} from "@al/ui/models/logData";
+import LogData, { type JSONLog } from "@al/ui/models/logData";
 import normalizer from "@al/ui/services/normalizer";
 
+interface IGroupedMsg {
+  msg: string;
+  logsCount: number;
+  hasErrors: boolean;
+  firstTime: string;
+  lastTime: string;
+}
+
+interface ISummaryMapGeneric<TGroupedMsg> {
+  msgs: Map<string, TGroupedMsg>;
+  httpCodes: Map<string, TGroupedMsg>;
+  jobs: Map<string, TGroupedMsg>;
+  plugins: Map<string, TGroupedMsg>;
+}
+
+type ISummaryMap = ISummaryMapGeneric<IGroupedMsg>;
+
 interface ITask {
   filePath: string;
 }
@@ -14,34 +28,36 @@ interface IResult {
   minTime: string;
   maxTime: string;
   size: number;
-  dataMap: SummaryMap;
+  dataMap: ISummaryMap;
 }
 
 // Thread Code
 if (parentPort) {
   parentPort.on("message", async (task: ITask) => {
     const stats = await processFile(task);
     parentPort!.postMessage(stats);
-    console.log(`Processed File: ${task.filePath}`);
+    console.log(`Processed File: ${stats.filePath}`);
   });
 }
 
 async function processFile(task: ITask): Promise<IResult> {
   const stats: IResult = {
-    filePath: task.filePath,
+    filePath: "",
     maxTime: "0",
     minTime: "z",
     size: 0,
     dataMap: {
-      httpCodes: new Map<string, GroupedMsg>(),
-      jobs: new Map<string, GroupedMsg>(),
-      msgs: new Map<string, GroupedMsg>(),
-      plugins: new Map<string, GroupedMsg>(),
+      httpCodes: new Map<string, IGroupedMsg>(),
+      jobs: new Map<string, IGroupedMsg>(),
+      msgs: new Map<string, IGroupedMsg>(),
+      plugins: new Map<string, IGroupedMsg>(),
     },
   };
 
   const logFile = Bun.file(task.filePath);
   stats.size = logFile.size;
+  // Keep only the last folder from the filePath
+  stats.filePath = task.filePath.split("/").slice(-2).join("/");
 
   const text = await logFile.text();
 
@@ -54,18 +70,60 @@ async function processFile(task: ITask): Promise<IResult> {
   for (const jsonLog of logsGeneratorFn()) {
     if (!jsonLog) continue;
 
-    if (jsonLog[LogData.logKeys.timestamp] > stats.maxTime) {
-      stats.maxTime = jsonLog[LogData.logKeys.timestamp];
+    const time = jsonLog[LogData.logKeys.timestamp];
+
+    if (time < stats.minTime) {
+      stats.minTime = time;
     }
 
-    if (jsonLog[LogData.logKeys.timestamp] < stats.minTime) {
-      stats.minTime = jsonLog[LogData.logKeys.timestamp];
+    if (time > stats.maxTime) {
+      stats.maxTime = time;
     }
 
-    LogData.initSummaryMap(jsonLog, stats.dataMap, false);
+    initSummaryMap(jsonLog, stats.dataMap);
   }
 
   return stats;
 }
 
-export type { ITask, IResult };
+function initSummaryMap(log: JSONLog, summaryMap: ISummaryMap) {
+  populateSummaryMap(log, summaryMap.msgs, LogData.msgKeySelector);
+  populateSummaryMap(log, summaryMap.jobs, LogData.jobKeySelector);
+  populateSummaryMap(log, summaryMap.httpCodes, LogData.httpCodeKeySelector);
+  populateSummaryMap(log, summaryMap.plugins, LogData.pluginKeySelector);
+}
+
+function populateSummaryMap(
+  log: JSONLog,
+  grpLogsMap: Map<string, IGroupedMsg>,
+  keySelectorFn: (log: JSONLog) => string | undefined
+) {
+  const key = keySelectorFn(log);
+  if (!key) return;
+
+  const time = log[LogData.logKeys.timestamp];
+
+  if (!grpLogsMap.has(key)) {
+    grpLogsMap.set(key, {
+      msg: key,
+      hasErrors: false,
+      logsCount: 0,
+      firstTime: time,
+      lastTime: time,
+    });
+  }
+
+  const grpLog = grpLogsMap.get(key)!;
+  grpLog.logsCount++;
+  grpLog.hasErrors ||= LogData.isErrorLog(log);
+
+  if (time < grpLog.firstTime) {
+    grpLog.firstTime = time;
+  }
+
+  if (time > grpLog.lastTime) {
+    grpLog.lastTime = time;
+  }
+}
+
+export type { ITask, IResult, IGroupedMsg, ISummaryMap, ISummaryMapGeneric };

ui/Taskfile.yml

@@ -16,3 +16,8 @@ tasks:
     cmds:
       - pnpm build
       - cp analog.sh analog.ps1 analog
+  
+  run:
+    desc: run
+    cmds:
+      - pnpm start

ui/src/components/filters/index.tsx

@@ -140,7 +140,9 @@ function Filters(props: FiltersProps) {
         <TextField
           label="Search"
           value={term.value}
-          onChange={(_, val) => setFilters("terms", i(), "value", val)}
+          onChange={(_, val) =>
+            setFilters("terms", i(), "value", val.toLowerCase())
+          }
           onKeyDown={handleEnterKey}
         />
       </>