Remove histograms from projects

Author: loicknuchel

frontend/src/Models/Project/ColumnDbStats.elm

@@ -34,7 +34,7 @@ encode value =
         , ( "bytesAvg", value.bytesAvg |> Encode.maybe Encode.float )
         , ( "cardinality", value.cardinality |> Encode.maybe Encode.float )
         , ( "commonValues", value.commonValues |> Encode.maybe (Encode.list encodeValue) )
-        , ( "histogram", value.histogram |> Encode.maybe (Encode.list Encode.string) )
+        , ( "histogram", Encode.null ) -- don't store histogram, bad chars can break Azimutt :/, old: value.histogram |> Encode.maybe (Encode.list Encode.string) )
         ]
 
 

libs/utils/src/array.test.ts

@@ -4,6 +4,7 @@ import {
     collectOne,
     diffBy,
     distinct,
+    duplicates,
     findLastIndex,
     groupBy,
     indexBy,
@@ -26,6 +27,9 @@ describe('array', () => {
     test('distinct', () => {
         expect(distinct([1, 1, 2, 3, 5, 3])).toEqual([1, 2, 3, 5])
     })
+    test('duplicates', () => {
+        expect(duplicates([1, 1, 2, 3, 5, 3, 1])).toEqual([1, 3])
+    })
     test('diffBy', () => {
         expect(diffBy(
             [{id: 1, name: 'a'}, {id: 2, name: 'b'}],

libs/utils/src/array.ts

@@ -33,6 +33,34 @@ export const distinctBy = <T>(arr: T[], by: (t: T) => string | number): T[] => {
     })
 }
 
+export const duplicates = <T>(arr: T[]): T[] => {
+    const seen = new Set<T>()
+    const dups = new Set<T>()
+    return arr.filter(t => {
+        if (seen.has(t)) {
+            if (dups.has(t)) return false
+            dups.add(t)
+            return true
+        }
+        seen.add(t)
+        return false
+    })
+}
+export const duplicatesBy = <T>(arr: T[], by: (t: T) => string | number): T[] => {
+    const seen = new Set<string | number>()
+    const dups = new Set<string | number>()
+    return arr.filter(t => {
+        const key = by(t)
+        if (seen.has(key)) {
+            if (dups.has(key)) return false
+            dups.add(key)
+            return true
+        }
+        seen.add(key)
+        return false
+    })
+}
+
 export type Diff<T> = {left: T[], right: T[], both: {left: T, right: T}[]}
 export const diffBy = <T>(arr1: T[], arr2: T[], f: (t: T, i: number) => string): Diff<T> => {
     const left: T[] = []

libs/utils/src/object.ts

@@ -1,8 +1,26 @@
-import {isEmpty} from "./validation";
 import {anySame} from "./any";
+import {Diff} from "./array";
+import {isEmpty} from "./validation";
 
 // functions sorted alphabetically
 
+export const objDiff = <T>(obj1: Record<string, T>, obj2: Record<string, T>): Diff<T> => {
+    const left: T[] = []
+    const both: {left: T, right: T}[] = []
+    const other = {...obj2}
+    Object.entries(obj1).forEach(([k, v1]) => {
+        const v2 = other[k]
+        if (v2 === undefined) {
+            left.push(v1)
+        } else {
+            both.push({left: v1, right: v2})
+        }
+        delete other[k]
+    })
+    const right: T[] = Object.values(other)
+    return {left, right, both}
+}
+
 export function equalDeep<T>(a: T, b: T): boolean {
     if (typeof a === 'string' || typeof a === 'number' || typeof a === 'boolean' || typeof a === 'symbol' || a === null || a === undefined) {
         return a === b
@@ -106,6 +124,26 @@ export function removeFieldsDeep(obj: any, keysToRemove: string[]): any {
     return obj
 }
 
+export function collectFieldsDeep(obj: any, keysToCollect: string[]): any[] {
+    if (Array.isArray(obj)) {
+        return obj.map(item => collectFieldsDeep(item, keysToCollect)).flat()
+    }
+
+    if (typeof obj === 'object' && obj !== null) {
+        const res: any[] = []
+        Object.keys(obj).forEach(key => {
+            if (keysToCollect.indexOf(key) >= 0) {
+                res.push(obj[key])
+            } else {
+                res.push(...collectFieldsDeep(obj[key], keysToCollect))
+            }
+        })
+        return res
+    }
+
+    return []
+}
+
 export function removeUndefined<K extends keyof any, V, T extends Record<K, V>>(obj: T): T {
     return filterValues(obj, v => v !== undefined)
 }