Skip to content

Commit 732a7a5

Browse files
marcelklehrmarcelklehr
authored
Merge pull request #2026 from floccusaddon/feat/murmur3
feat: Optionally use murmurhash3 instead of sha256
2 parents 1db577a + 8c88dad commit 732a7a5

File tree

20 files changed

+272
-64
lines changed

20 files changed

+272
-64
lines changed

‎src/lib/CachingTreeWrapper.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { CachingResource, OrderFolderResource } from './interfaces/Resource'
1+
import { CachingResource, ICapabilities, IHashSettings, OrderFolderResource } from './interfaces/Resource'
22
import { Bookmark, Folder, ItemLocation } from './Tree'
33
import CacheTree from './CacheTree'
44
import Ordering from './interfaces/Ordering'
@@ -74,4 +74,12 @@ export default class CachingTreeWrapper implements OrderFolderResource<typeof It
7474
getCacheTree(): Promise<Folder<typeof ItemLocation.LOCAL>> {
7575
return this.cacheTree.getBookmarksTree()
7676
}
77+
78+
getCapabilities(): Promise<ICapabilities> {
79+
return this.innerTree.getCapabilities()
80+
}
81+
82+
setHashSettings(hashSettings: IHashSettings): void {
83+
this.innerTree.setHashSettings(hashSettings)
84+
}
7785
}

‎src/lib/Crypto.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,17 @@
11
import { fromUint8Array, toUint8Array } from 'js-base64'
2+
import { murmurhash3_32_gc } from './murmurhash3'
23

34
export default class Crypto {
45
static iterations = 250000
56
static ivLength = 16
67

8+
static async murmurHash3(message: string): Promise<string> {
9+
const buf32 = new Uint32Array([murmurhash3_32_gc(message, 0)])
10+
const buf8 = new Uint8Array(buf32.buffer)
11+
buf8.reverse()
12+
return this.bufferToHexstr(buf8)
13+
}
14+
715
static async sha256(message: string): Promise<string> {
816
const msgBuffer = new TextEncoder().encode(message) // encode as UTF-8
917
const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer) // hash the message

‎src/lib/LocalTabs.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import browser from './browser-api'
22
import Logger from './Logger'
3-
import { OrderFolderResource } from './interfaces/Resource'
3+
import { ICapabilities, IHashSettings, OrderFolderResource } from './interfaces/Resource'
44
import PQueue from 'p-queue'
55
import { Bookmark, Folder, ItemLocation } from './Tree'
66
import Ordering from './interfaces/Ordering'
@@ -502,6 +502,18 @@ export default class LocalTabs implements OrderFolderResource<typeof ItemLocatio
502502
async isUsingBrowserTabs() {
503503
return true
504504
}
505+
506+
async getCapabilities(): Promise<ICapabilities> {
507+
return {
508+
preserveOrder: true,
509+
hashFn: ['murmur3', 'sha256']
510+
}
511+
}
512+
513+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
514+
setHashSettings(hashSettings: IHashSettings): void {
515+
// noop
516+
}
505517
}
506518

507519
function awaitTabsUpdated() {

‎src/lib/Scanner.ts‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import * as Parallel from 'async-parallel'
22
import Diff, { ActionType, CreateAction, MoveAction, RemoveAction, ReorderAction, UpdateAction } from './Diff'
33
import { Bookmark, Folder, ItemLocation, ItemType, TItem, TItemLocation } from './Tree'
44
import Logger from './Logger'
5+
import { IHashSettings } from './interfaces/Resource'
56

67
export interface ScanResult<L1 extends TItemLocation, L2 extends TItemLocation> {
78
CREATE: Diff<L1, L2, CreateAction<L1, L2>>
@@ -15,17 +16,17 @@ export default class Scanner<L1 extends TItemLocation, L2 extends TItemLocation>
1516
private oldTree: TItem<L1>
1617
private newTree: TItem<L2>
1718
private mergeable: (i1: TItem<TItemLocation>, i2: TItem<TItemLocation>) => boolean
18-
private preserveOrder: boolean
19+
private hashSettings: IHashSettings
1920
private checkHashes: boolean
2021
private hasCache: boolean
2122

2223
private result: ScanResult<L2, L1>
2324

24-
constructor(oldTree:TItem<L1>, newTree:TItem<L2>, mergeable:(i1:TItem<TItemLocation>, i2:TItem<TItemLocation>)=>boolean, preserveOrder:boolean, checkHashes = true, hasCache = true) {
25+
constructor(oldTree:TItem<L1>, newTree:TItem<L2>, mergeable:(i1:TItem<TItemLocation>, i2:TItem<TItemLocation>)=>boolean, hashSettings: IHashSettings, checkHashes = true, hasCache = true) {
2526
this.oldTree = oldTree
2627
this.newTree = newTree
2728
this.mergeable = mergeable
28-
this.preserveOrder = preserveOrder
29+
this.hashSettings = hashSettings
2930
this.checkHashes = typeof checkHashes === 'undefined' ? true : checkHashes
3031
this.hasCache = hasCache
3132
this.result = {
@@ -136,14 +137,14 @@ export default class Scanner<L1 extends TItemLocation, L2 extends TItemLocation>
136137
}
137138

138139
async bookmarkHasChanged(oldBookmark:Bookmark<L1>, newBookmark:Bookmark<L2>):Promise<boolean> {
139-
const oldHash = await oldBookmark.hash()
140-
const newHash = await newBookmark.hash()
140+
const oldHash = await oldBookmark.hash(this.hashSettings)
141+
const newHash = await newBookmark.hash(this.hashSettings)
141142
return oldHash !== newHash
142143
}
143144

144145
async folderHasChanged(oldFolder:Folder<L1>, newFolder:Folder<L2>):Promise<boolean> {
145-
const oldHash = await oldFolder.hash(this.preserveOrder)
146-
const newHash = await newFolder.hash(this.preserveOrder)
146+
const oldHash = await oldFolder.hash(this.hashSettings)
147+
const newHash = await newFolder.hash(this.hashSettings)
147148
return oldHash !== newHash
148149
}
149150

‎src/lib/Tree.ts‎

Lines changed: 46 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import Crypto from './Crypto'
22
import Logger from './Logger'
3-
import TResource from './interfaces/Resource'
3+
import TResource, { IHashSettings } from './interfaces/Resource'
44
import * as Parallel from 'async-parallel'
55

66
const STRANGE_PROTOCOLS = ['data:', 'javascript:', 'about:', 'chrome:', 'file:']
@@ -37,7 +37,7 @@ export class Bookmark<L extends TItemLocation> {
3737
public tags: string[]
3838
public location: L
3939
public isRoot = false
40-
private hashValue: string
40+
private hashValue: Record<string, string>
4141

4242
constructor({ id, parentId, url, title, tags, location }: { id:string|number, parentId:string|number, url:string, title:string, tags?: string[], location: L }) {
4343
this.id = id
@@ -78,13 +78,25 @@ export class Bookmark<L extends TItemLocation> {
7878
return 0
7979
}
8080

81-
async hash():Promise<string> {
81+
setHashCacheValue(hashSettings: IHashSettings, value: string): void {
82+
const cacheKey = `${hashSettings.preserveOrder}-${hashSettings.hashFn}`
83+
if (!this.hashValue) this.hashValue = {}
84+
this.hashValue[cacheKey] = value
85+
}
86+
87+
async hash({preserveOrder = false, hashFn = 'sha256'}):Promise<string> {
8288
if (!this.hashValue) {
83-
this.hashValue = await Crypto.sha256(
84-
JSON.stringify({ title: this.title, url: this.url })
85-
)
89+
this.hashValue = {}
90+
const json = JSON.stringify({ title: this.title, url: this.url })
91+
if (hashFn === 'sha256') {
92+
this.hashValue[hashFn] = await Crypto.sha256(json)
93+
} else if (hashFn === 'murmur3') {
94+
this.hashValue[hashFn] = await Crypto.murmurHash3(json)
95+
} else {
96+
throw new Error('Unsupported hash function specified')
97+
}
8698
}
87-
return this.hashValue
99+
return this.hashValue[hashFn]
88100
}
89101

90102
clone(withHash?: boolean):Bookmark<L> {
@@ -119,6 +131,7 @@ export class Bookmark<L extends TItemLocation> {
119131
toJSON() {
120132
// Flatten inherited properties for serialization
121133
const result = {}
134+
// eslint-disable-next-line @typescript-eslint/no-this-alias
122135
let obj = this
123136
while (obj) {
124137
Object.entries(obj).forEach(([key, value]) => {
@@ -306,9 +319,16 @@ export class Folder<L extends TItemLocation> {
306319
return 0
307320
}
308321

309-
async hash(preserveOrder = false): Promise<string> {
310-
if (this.hashValue && typeof this.hashValue[String(preserveOrder)] !== 'undefined') {
311-
return this.hashValue[String(preserveOrder)]
322+
setHashCacheValue(hashSettings: IHashSettings, value: string): void {
323+
const cacheKey = `${hashSettings.preserveOrder}-${hashSettings.hashFn}`
324+
if (!this.hashValue) this.hashValue = {}
325+
this.hashValue[cacheKey] = value
326+
}
327+
328+
async hash({preserveOrder = false, hashFn = 'sha256'}: IHashSettings = {preserveOrder: false, hashFn: 'sha256'}): Promise<string> {
329+
const cacheKey = `${preserveOrder}-${hashFn}`
330+
if (this.hashValue && typeof this.hashValue[cacheKey] !== 'undefined') {
331+
return this.hashValue[cacheKey]
312332
}
313333

314334
if (!this.loaded) {
@@ -329,17 +349,22 @@ export class Folder<L extends TItemLocation> {
329349
})
330350
}
331351
if (!this.hashValue) this.hashValue = {}
332-
this.hashValue[String(preserveOrder)] = await Crypto.sha256(
333-
JSON.stringify({
334-
title: this.title,
335-
children: await Parallel.map(
336-
children,
337-
child => child.hash(preserveOrder),
338-
1
339-
)
340-
})
341-
)
342-
return this.hashValue[String(preserveOrder)]
352+
const json = JSON.stringify({
353+
title: this.title,
354+
children: await Parallel.map(
355+
children,
356+
child => child.hash({preserveOrder, hashFn}),
357+
1
358+
)
359+
})
360+
if (hashFn === 'sha256') {
361+
this.hashValue[cacheKey] = await Crypto.sha256(json)
362+
} else if (hashFn === 'murmur3') {
363+
this.hashValue[cacheKey] = await Crypto.murmurHash3(json)
364+
} else {
365+
throw new Error('Unsupported hash function specified')
366+
}
367+
return this.hashValue[cacheKey]
343368
}
344369

345370
copy(withHash?:boolean):Folder<L> {

‎src/lib/adapters/Caching.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,14 @@ import {
1212
UnknownMoveOriginError,
1313
UnknownMoveTargetError
1414
} from '../../errors/Error'
15-
import { BulkImportResource } from '../interfaces/Resource'
15+
import { BulkImportResource, ICapabilities, IHashSettings } from '../interfaces/Resource'
1616

1717
export default class CachingAdapter implements Adapter, BulkImportResource<TItemLocation> {
1818
protected highestId: number
1919
public bookmarksCache: Folder<TItemLocation>
2020
protected server: any
2121
protected location: TItemLocation = ItemLocation.SERVER
22+
protected hashSettings: IHashSettings
2223

2324
constructor(server: any) {
2425
this.resetCache()
@@ -245,4 +246,15 @@ export default class CachingAdapter implements Adapter, BulkImportResource<TItem
245246
isAvailable(): Promise<boolean> {
246247
return Promise.resolve(true)
247248
}
249+
250+
async getCapabilities(): Promise<ICapabilities> {
251+
return {
252+
preserveOrder: true,
253+
hashFn: ['murmur3', 'sha256'],
254+
}
255+
}
256+
257+
setHashSettings(hashSettings: IHashSettings): void {
258+
this.hashSettings = hashSettings
259+
}
248260
}

‎src/lib/adapters/Git.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ export default class GitAdapter extends CachingAdapter {
170170

171171
const status = await this.pullFromServer()
172172

173-
this.initialTreeHash = await this.bookmarksCache.hash(true)
173+
this.initialTreeHash = await this.bookmarksCache.hash(this.hashSettings)
174174

175175
Logger.log('onSyncStart: completed')
176176

@@ -189,7 +189,7 @@ export default class GitAdapter extends CachingAdapter {
189189
clearInterval(this.lockingInterval)
190190

191191
this.bookmarksCache = this.bookmarksCache.clone(false)
192-
const newTreeHash = await this.bookmarksCache.hash(true)
192+
const newTreeHash = await this.bookmarksCache.hash(this.hashSettings)
193193
if (newTreeHash !== this.initialTreeHash) {
194194
const fileContents = this.server.bookmark_file_type === 'xbel' ? createXBEL(this.bookmarksCache, this.highestId) : createHTML(this.bookmarksCache, this.highestId)
195195
Logger.log('(git) writeFile ' + this.dir + '/' + this.server.bookmark_file)

‎src/lib/adapters/GoogleDrive.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,7 @@ export default class GoogleDriveAdapter extends CachingAdapter {
298298
this.alwaysUpload = true
299299
}
300300

301-
this.initialTreeHash = await this.bookmarksCache.hash(true)
301+
this.initialTreeHash = await this.bookmarksCache.hash(this.hashSettings)
302302

303303
Logger.log('onSyncStart: completed')
304304

@@ -335,7 +335,7 @@ export default class GoogleDriveAdapter extends CachingAdapter {
335335
clearInterval(this.lockingInterval)
336336

337337
this.bookmarksCache = this.bookmarksCache.clone(false)
338-
const newTreeHash = await this.bookmarksCache.hash(true)
338+
const newTreeHash = await this.bookmarksCache.hash(this.hashSettings)
339339

340340
if (!this.fileId) {
341341
await this.createFile(await this.getXBELContent())

‎src/lib/adapters/Karakeep.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import Adapter from '../interfaces/Adapter'
22
import { Bookmark, Folder, ItemLocation } from '../Tree'
33
import PQueue from 'p-queue'
4-
import { IResource } from '../interfaces/Resource'
4+
import { ICapabilities, IHashSettings, IResource } from '../interfaces/Resource'
55
import Logger from '../Logger'
66
import {
77
AuthenticationError,
@@ -565,4 +565,16 @@ export default class KarakeepAdapter implements Adapter, IResource<typeof ItemLo
565565

566566
return json
567567
}
568+
569+
async getCapabilities(): Promise<ICapabilities> {
570+
return {
571+
preserveOrder: false,
572+
hashFn: ['murmur3', 'sha256'],
573+
}
574+
}
575+
576+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
577+
setHashSettings(hashSettings: IHashSettings): void {
578+
// noop
579+
}
568580
}

‎src/lib/adapters/Linkwarden.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import Adapter from '../interfaces/Adapter'
22
import { Bookmark, Folder, ItemLocation } from '../Tree'
33
import PQueue from 'p-queue'
4-
import { IResource } from '../interfaces/Resource'
4+
import { ICapabilities, IHashSettings, IResource } from '../interfaces/Resource'
55
import Logger from '../Logger'
66
import {
77
AuthenticationError,
@@ -367,4 +367,16 @@ export default class LinkwardenAdapter implements Adapter, IResource<typeof Item
367367

368368
return json
369369
}
370+
371+
async getCapabilities(): Promise<ICapabilities> {
372+
return {
373+
preserveOrder: false,
374+
hashFn: ['murmur3', 'sha256'],
375+
}
376+
}
377+
378+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
379+
setHashSettings(hashSettings: IHashSettings): void {
380+
// noop
381+
}
370382
}

0 commit comments

Comments
 (0)