Skip to content

Commit d1bfc94

Browse files
achingbrainachingbrain
authored
fix: use uint8arrays alloc for new buffers (#123)
Will return `Buffer`s under Node.js which have greater performance than `Uint8Array`s.
1 parent 23073eb commit d1bfc94

File tree

16 files changed

+163
-85
lines changed

16 files changed

+163
-85
lines changed

‎packages/protons/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@
139139
"pbjs": "^0.0.14",
140140
"protobufjs": "^7.0.0",
141141
"protons-runtime": "^5.0.0",
142-
"uint8arraylist": "^2.4.3"
142+
"uint8arraylist": "^2.4.3",
143+
"uint8arrays": "^4.0.6"
143144
}
144145
}

‎packages/protons/src/index.ts‎

Lines changed: 122 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ const decoderGenerators: Record<string, (jsTypeOverride?: 'number' | 'string') =
190190

191191
const defaultValueGenerators: Record<string, () => string> = {
192192
bool: () => 'false',
193-
bytes: () => 'new Uint8Array(0)',
193+
bytes: () => 'uint8ArrayAlloc(0)',
194194
double: () => '0',
195195
fixed32: () => '0',
196196
fixed64: () => '0n',
@@ -320,6 +320,10 @@ function createDefaultObject (fields: Record<string, FieldDef>, messageDef: Mess
320320
defaultValueGenerator = defaultValueGeneratorsJsTypeOverrides[jsTypeOverride]
321321
}
322322

323+
if (type === 'bytes') {
324+
moduleDef.addImport('uint8arrays/alloc', 'alloc', 'uint8ArrayAlloc')
325+
}
326+
323327
defaultValue = defaultValueGenerator()
324328
} else {
325329
const def = findDef(fieldDef.type, messageDef, moduleDef)
@@ -457,7 +461,7 @@ function defineFields (fields: Record<string, FieldDef>, messageDef: MessageDef,
457461

458462
function compileMessage (messageDef: MessageDef, moduleDef: ModuleDef, flags?: Flags): string {
459463
if (isEnumDef(messageDef)) {
460-
moduleDef.imports.add('enumeration')
464+
moduleDef.addImport('protons-runtime', 'enumeration')
461465

462466
// check that the enum def values start from 0
463467
if (Object.values(messageDef.values)[0] !== 0) {
@@ -510,10 +514,11 @@ export namespace ${messageDef.name} {
510514
const fields = messageDef.fields ?? {}
511515

512516
// import relevant modules
513-
moduleDef.imports.add('encodeMessage')
514-
moduleDef.imports.add('decodeMessage')
515-
moduleDef.imports.add('message')
516-
moduleDef.importedTypes.add('Codec')
517+
moduleDef.addImport('protons-runtime', 'encodeMessage')
518+
moduleDef.addImport('protons-runtime', 'decodeMessage')
519+
moduleDef.addImport('protons-runtime', 'message')
520+
moduleDef.addTypeImport('protons-runtime', 'Codec')
521+
moduleDef.addTypeImport('uint8arraylist', 'Uint8ArrayList')
517522

518523
const interfaceFields = defineFields(fields, messageDef, moduleDef)
519524
.join('\n ')
@@ -544,10 +549,10 @@ export interface ${messageDef.name} {
544549

545550
if (codec == null) {
546551
if (fieldDef.enum) {
547-
moduleDef.imports.add('enumeration')
552+
moduleDef.addImport('protons-runtime', 'enumeration')
548553
type = 'enum'
549554
} else {
550-
moduleDef.imports.add('message')
555+
moduleDef.addImport('protons-runtime', 'message')
551556
type = 'message'
552557
}
553558

@@ -669,10 +674,10 @@ export interface ${messageDef.name} {
669674

670675
if (codec == null) {
671676
if (fieldDef.enum) {
672-
moduleDef.imports.add('enumeration')
677+
moduleDef.addImport('protons-runtime', 'enumeration')
673678
type = 'enum'
674679
} else {
675-
moduleDef.imports.add('message')
680+
moduleDef.addImport('protons-runtime', 'message')
676681
type = 'message'
677682
}
678683

@@ -689,7 +694,7 @@ export interface ${messageDef.name} {
689694
let limit = ''
690695

691696
if (fieldDef.lengthLimit != null) {
692-
moduleDef.imports.add('CodeError')
697+
moduleDef.addImport('protons-runtime', 'CodeError')
693698

694699
limit = `
695700
if (obj.${fieldName}.size === ${fieldDef.lengthLimit}) {
@@ -707,7 +712,7 @@ export interface ${messageDef.name} {
707712
let limit = ''
708713

709714
if (fieldDef.lengthLimit != null) {
710-
moduleDef.imports.add('CodeError')
715+
moduleDef.addImport('protons-runtime', 'CodeError')
711716

712717
limit = `
713718
if (obj.${fieldName}.length === ${fieldDef.lengthLimit}) {
@@ -785,23 +790,83 @@ export namespace ${messageDef.name} {
785790
`.trimStart()
786791
}
787792

788-
interface ModuleDef {
789-
imports: Set<string>
790-
importedTypes: Set<string>
793+
interface Import {
794+
symbol: string
795+
alias?: string
796+
type: boolean
797+
}
798+
799+
class ModuleDef {
800+
imports: Map<string, Import[]>
791801
types: Set<string>
792802
compiled: string[]
793803
globals: Record<string, ClassDef>
794-
}
795804

796-
function defineModule (def: ClassDef, flags: Flags): ModuleDef {
797-
const moduleDef: ModuleDef = {
798-
imports: new Set(),
799-
importedTypes: new Set(),
800-
types: new Set(),
801-
compiled: [],
802-
globals: {}
805+
constructor () {
806+
this.imports = new Map()
807+
this.types = new Set()
808+
this.compiled = []
809+
this.globals = {}
810+
}
811+
812+
addImport (module: string, symbol: string, alias?: string): void {
813+
const defs = this._findDefs(module)
814+
815+
for (const def of defs) {
816+
// check if we already have a definition for this symbol
817+
if (def.symbol === symbol) {
818+
if (alias !== def.alias) {
819+
throw new Error(`Type symbol ${symbol} imported from ${module} with alias ${def.alias} does not match alias ${alias}`)
820+
}
821+
822+
// if it was a type before it's not now
823+
def.type = false
824+
return
825+
}
826+
}
827+
828+
defs.push({
829+
symbol,
830+
alias,
831+
type: false
832+
})
833+
}
834+
835+
addTypeImport (module: string, symbol: string, alias?: string): void {
836+
const defs = this._findDefs(module)
837+
838+
for (const def of defs) {
839+
// check if we already have a definition for this symbol
840+
if (def.symbol === symbol) {
841+
if (alias !== def.alias) {
842+
throw new Error(`Type symbol ${symbol} imported from ${module} with alias ${def.alias} does not match alias ${alias}`)
843+
}
844+
845+
return
846+
}
847+
}
848+
849+
defs.push({
850+
symbol,
851+
alias,
852+
type: true
853+
})
803854
}
804855

856+
_findDefs (module: string): Import[] {
857+
let defs = this.imports.get(module)
858+
859+
if (defs == null) {
860+
defs = []
861+
this.imports.set(module, defs)
862+
}
863+
864+
return defs
865+
}
866+
}
867+
868+
function defineModule (def: ClassDef, flags: Flags): ModuleDef {
869+
const moduleDef = new ModuleDef()
805870
const defs = def.nested
806871

807872
if (defs == null) {
@@ -963,28 +1028,48 @@ export async function generate (source: string, flags: Flags): Promise<void> {
9631028
]
9641029

9651030
const imports = []
1031+
const importedModules = Array.from([...moduleDef.imports.entries()])
1032+
.sort((a, b) => {
1033+
return a[0].localeCompare(b[0])
1034+
})
1035+
.sort((a, b) => {
1036+
const aAllTypes = a[1].reduce((acc, curr) => {
1037+
return acc && curr.type
1038+
}, true)
9661039

967-
if (moduleDef.imports.size > 0) {
968-
imports.push(`import { ${Array.from(moduleDef.imports).join(', ')} } from 'protons-runtime'`)
969-
}
1040+
const bAllTypes = b[1].reduce((acc, curr) => {
1041+
return acc && curr.type
1042+
}, true)
9701043

971-
if (moduleDef.imports.has('encodeMessage')) {
972-
imports.push("import type { Uint8ArrayList } from 'uint8arraylist'")
973-
}
1044+
if (aAllTypes && !bAllTypes) {
1045+
return 1
1046+
}
1047+
1048+
if (!aAllTypes && bAllTypes) {
1049+
return -1
1050+
}
9741051

975-
if (moduleDef.importedTypes.size > 0) {
976-
imports.push(`import type { ${Array.from(moduleDef.importedTypes).join(', ')} } from 'protons-runtime'`)
1052+
return 0
1053+
})
1054+
1055+
for (const imp of importedModules) {
1056+
const allTypes = imp[1].reduce((acc, curr) => {
1057+
return acc && curr.type
1058+
}, true)
1059+
1060+
const symbols = imp[1].sort((a, b) => {
1061+
return a.symbol.localeCompare(b.symbol)
1062+
}).map(imp => {
1063+
return `${!allTypes && imp.type ? 'type ' : ''}${imp.symbol}${imp.alias != null ? ` as ${imp.alias}` : ''}`
1064+
}).join(', ')
1065+
1066+
imports.push(`import ${allTypes ? 'type ' : ''}{ ${symbols} } from '${imp[0]}'`)
9771067
}
9781068

9791069
const lines = [
9801070
...ignores,
9811071
'',
982-
...imports.sort((a, b) => {
983-
const aModule = a.split("from '")[1].toString()
984-
const bModule = b.split("from '")[1].toString()
985-
986-
return aModule.localeCompare(bModule)
987-
}),
1072+
...imports,
9881073
'',
9891074
...moduleDef.compiled
9901075
]

‎packages/protons/test/fixtures/basic.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@
44
/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
55
/* eslint-disable @typescript-eslint/no-empty-interface */
66

7-
import { encodeMessage, decodeMessage, message } from 'protons-runtime'
8-
import type { Codec } from 'protons-runtime'
7+
import { type Codec, decodeMessage, encodeMessage, message } from 'protons-runtime'
98
import type { Uint8ArrayList } from 'uint8arraylist'
109

1110
export interface Basic {

‎packages/protons/test/fixtures/bitswap.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
55
/* eslint-disable @typescript-eslint/no-empty-interface */
66

7-
import { enumeration, encodeMessage, decodeMessage, message } from 'protons-runtime'
8-
import type { Codec } from 'protons-runtime'
7+
import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime'
8+
import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc'
99
import type { Uint8ArrayList } from 'uint8arraylist'
1010

1111
export interface Message {
@@ -87,7 +87,7 @@ export namespace Message {
8787
}
8888
}, (reader, length) => {
8989
const obj: any = {
90-
block: new Uint8Array(0),
90+
block: uint8ArrayAlloc(0),
9191
priority: 0,
9292
wantType: WantType.Block,
9393
sendDontHave: false
@@ -239,8 +239,8 @@ export namespace Message {
239239
}
240240
}, (reader, length) => {
241241
const obj: any = {
242-
prefix: new Uint8Array(0),
243-
data: new Uint8Array(0)
242+
prefix: uint8ArrayAlloc(0),
243+
data: uint8ArrayAlloc(0)
244244
}
245245

246246
const end = length == null ? reader.len : reader.pos + length
@@ -326,7 +326,7 @@ export namespace Message {
326326
}
327327
}, (reader, length) => {
328328
const obj: any = {
329-
cid: new Uint8Array(0),
329+
cid: uint8ArrayAlloc(0),
330330
type: BlockPresenceType.Have
331331
}
332332

‎packages/protons/test/fixtures/circuit.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
55
/* eslint-disable @typescript-eslint/no-empty-interface */
66

7-
import { enumeration, encodeMessage, decodeMessage, message } from 'protons-runtime'
8-
import type { Codec } from 'protons-runtime'
7+
import { type Codec, decodeMessage, encodeMessage, enumeration, message } from 'protons-runtime'
8+
import { alloc as uint8ArrayAlloc } from 'uint8arrays/alloc'
99
import type { Uint8ArrayList } from 'uint8arraylist'
1010

1111
export interface CircuitRelay {
@@ -112,7 +112,7 @@ export namespace CircuitRelay {
112112
}
113113
}, (reader, length) => {
114114
const obj: any = {
115-
id: new Uint8Array(0),
115+
id: uint8ArrayAlloc(0),
116116
addrs: []
117117
}
118118

‎packages/protons/test/fixtures/custom-option-jstype.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@
44
/* eslint-disable @typescript-eslint/no-unnecessary-boolean-literal-compare */
55
/* eslint-disable @typescript-eslint/no-empty-interface */
66

7-
import { encodeMessage, decodeMessage, message } from 'protons-runtime'
8-
import type { Codec } from 'protons-runtime'
7+
import { type Codec, decodeMessage, encodeMessage, message } from 'protons-runtime'
98
import type { Uint8ArrayList } from 'uint8arraylist'
109

1110
export interface CustomOptionNumber {

0 commit comments

Comments
 (0)