Skip to content

Commit 7927097

Browse files
committed
tolk-js v0.6 implementation
tolk-js is conceptually similar to func-js. But unlike func-js, which is separated into two packages (func-js-bin), tolk-js contains API and WASM here, in a single place. Its versioning will be maintained equal to a Tolk Compiler, since it's an essential part of distribution. Historically, tolk-js was developed for several months, in sync with the development of the Tolk compiler. But for the first public release, that dev history is not needed. That's why I just do a single giant commit.
1 parent 7385311 commit 7927097

File tree

14 files changed

+2842
-0
lines changed

14 files changed

+2842
-0
lines changed

jest.config.js

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/** @type {import('ts-jest').JestConfigWithTsJest} **/
2+
module.exports = {
3+
testEnvironment: "node",
4+
transform: {
5+
"^.+.tsx?$": ["ts-jest",{}],
6+
},
7+
};

package.json

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
{
2+
"name": "@ton/tolk-js",
3+
"version": "0.6.0",
4+
"description": "Tolk Language compiler (next-generation FunC)",
5+
"main": "dist/index.js",
6+
"bin": "./dist/cli.js",
7+
"files": [
8+
"dist/*"
9+
],
10+
"scripts": {
11+
"build": "rm -rf dist && tsc && yarn wasm:pack && yarn stdlib:pack && yarn wasm:dist && yarn stdlib:dist",
12+
"wasm:pack": "node scripts/pack-wasm-to-base64.js",
13+
"stdlib:pack": "node scripts/pack-stdlib-to-object.js",
14+
"wasm:dist": "cp src/tolkfiftlib.js dist && cp src/tolkfiftlib.wasm.js dist",
15+
"stdlib:dist": "cp -r src/tolk-stdlib dist && cp src/stdlib.tolk.js dist",
16+
"test": "yarn wasm:pack && yarn stdlib:pack && yarn jest"
17+
},
18+
"author": "TON Blockchain",
19+
"license": "MIT",
20+
"repository": {
21+
"type": "git",
22+
"url": "git+https://github.com/ton-blockchain/tolk-js.git"
23+
},
24+
"devDependencies": {
25+
"@ton/core": "^0.56.3",
26+
"@ton/crypto": "^3.3.0",
27+
"@types/jest": "^29.5.12",
28+
"jest": "^29.7.0",
29+
"ts-jest": "^29.2.4",
30+
"typescript": "^5.5.4"
31+
},
32+
"dependencies": {
33+
"arg": "^5.0.2"
34+
}
35+
}

scripts/pack-stdlib-to-object.js

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
const fs = require('fs')
2+
3+
let out = `// stdlib files are stored as strings in order to make it work on web (without fs.readFile)
4+
module.exports = {\n\n`
5+
6+
let fileNames = [
7+
...fs.readdirSync('./src/tolk-stdlib').filter(s => s.endsWith('.tolk')).sort(),
8+
]
9+
10+
for (let fileName of fileNames) {
11+
const contents = fs.readFileSync('./src/tolk-stdlib/' + fileName, 'utf-8')
12+
out += `'@stdlib/${fileName}':\`${contents.replace(/`/g, '\\`')}\`,\n\n`
13+
}
14+
15+
out += "};\n"
16+
fs.writeFileSync('./src/stdlib.tolk.js', out)
17+
18+
// note, that Asm.fif and similar are embedded into wasm binary,
19+
// but stdlib files are embedded here and distributed like separate files also -
20+
// they are for IDE purposes, since both plugins for VS Code and IDEA auto-locate these files

scripts/pack-wasm-to-base64.js

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
const fs = require('fs')
2+
3+
const wasmBinary = fs.readFileSync('./src/tolkfiftlib.wasm')
4+
const out = `// tolkfiftlib.wasm is packed to base64 in order to make it work on web (without fs.readFile)
5+
module.exports = '${wasmBinary.toString('base64')}';`
6+
7+
fs.writeFileSync('./src/tolkfiftlib.wasm.js', out)

src/cli.ts

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#!/usr/bin/env node
2+
import arg from 'arg'
3+
import fs from 'fs'
4+
import path from 'path'
5+
import {runTolkCompiler, getTolkCompilerVersion} from '.'
6+
7+
async function tolkJsCli() {
8+
const args = arg({
9+
'--version': Boolean,
10+
'--help': Boolean,
11+
'--require-version': String,
12+
'--output-json': String,
13+
'--output-fift': String,
14+
'--experimental-options': String,
15+
'--cwd': String,
16+
17+
'-v': '--version',
18+
'-h': '--help',
19+
'-o': '--output-json',
20+
'-x': '--experimental-options',
21+
'-C': '--cwd',
22+
})
23+
24+
if (args['--help']) {
25+
console.log(`Usage: tolk-js [OPTIONS] entrypointFileName.tolk
26+
Options:
27+
-h, --help — print this help and exit
28+
-v, --version — print Tolk compiler version and exit
29+
--require-version <version> — exit if Tolk compiler version differs from the required
30+
--output-json <filename>, -o <filename> — output .json file with BoC, Fift code, and some other stuff
31+
--output-fif <filename> - output .fif file with Fift code output
32+
--experimental-options <names> - set experimental compiler options, comma-separated
33+
--cwd <path>, -C <path> — sets cwd to locate .tolk files (doesn't affect output paths)
34+
`)
35+
process.exit(0)
36+
}
37+
38+
const version = await getTolkCompilerVersion()
39+
40+
if (args['--version']) {
41+
console.log(`Tolk compiler v${version}`)
42+
process.exit(0)
43+
}
44+
45+
if (args['--require-version'] !== undefined && version !== args['--require-version']) {
46+
throw `Failed to run tolk-js: --require-version = ${args['--require-version']}, but Tolk compiler version = ${version}`
47+
}
48+
49+
if (args._.length !== 1) {
50+
throw 'entrypointFileName wasn\'t specified. Run with -h to see help.'
51+
}
52+
53+
console.log(`Compiling using Tolk v${version}`)
54+
55+
const cwd = args['--cwd']
56+
const result = await runTolkCompiler({
57+
entrypointFileName: args._[0],
58+
experimentalOptions: args['--experimental-options'],
59+
fsReadCallback: p => fs.readFileSync(cwd ? path.join(cwd, p) : p, 'utf-8'),
60+
})
61+
62+
if (result.status === 'error') {
63+
throw result.message
64+
}
65+
66+
if (args['--output-json']) {
67+
fs.writeFileSync(args['--output-json'], JSON.stringify({
68+
artifactVersion: 1,
69+
tolkVersion: version,
70+
fiftCode: result.fiftCode,
71+
codeBoc64: result.codeBoc64,
72+
codeHashHex: result.codeHashHex,
73+
sourcesSnapshot: result.sourcesSnapshot,
74+
}, null, 2))
75+
}
76+
77+
if (args['--output-fift']) {
78+
fs.writeFileSync(args['--output-fift'], result.fiftCode)
79+
}
80+
81+
console.log('Compiled successfully!')
82+
83+
if (!args['--output-json'] && !args['--output-fift']) {
84+
console.warn('Warning: No output options were specified. Run with -h to see help.')
85+
} else {
86+
console.log('Written output files.')
87+
}
88+
}
89+
90+
tolkJsCli().catch(ex => {
91+
console.error(ex)
92+
process.exit(1)
93+
})

src/index.ts

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
// @ts-ignore
2+
import wasmModule from "./tolkfiftlib.js"
3+
// @ts-ignore
4+
import wasmBase64 from "./tolkfiftlib.wasm.js"
5+
// @ts-ignore
6+
import stdlibContents from "./stdlib.tolk.js"
7+
import {realpath} from "./path-utils"
8+
9+
let wasmBinary: Uint8Array | undefined = undefined
10+
11+
type WasmModule = any
12+
13+
export type FsReadCallback = (path: string) => string
14+
15+
export type TolkCompilerConfig = {
16+
entrypointFileName: string
17+
fsReadCallback: FsReadCallback
18+
optimizationLevel?: number
19+
withStackComments?: boolean
20+
experimentalOptions?: string
21+
}
22+
23+
export type TolkResultSuccess = {
24+
status: "ok"
25+
fiftCode: string
26+
codeBoc64: string
27+
codeHashHex: string
28+
stderr: string
29+
sourcesSnapshot: { filename: string, contents: string }[]
30+
}
31+
32+
export type TolkResultError = {
33+
status: "error"
34+
message: string
35+
}
36+
37+
function copyToCStringAllocating(mod: WasmModule, inStr: string): any {
38+
const len = mod.lengthBytesUTF8(inStr) + 1
39+
const ptr = mod._malloc(len)
40+
mod.stringToUTF8(inStr, ptr, len)
41+
return ptr
42+
}
43+
44+
function copyToCStringPtr(mod: WasmModule, inStr: string, destPtr: any): any {
45+
const allocated = copyToCStringAllocating(mod, inStr)
46+
mod.setValue(destPtr, allocated, '*')
47+
return allocated
48+
}
49+
50+
function copyFromCString(mod: WasmModule, inPtr: any): string {
51+
return mod.UTF8ToString(inPtr)
52+
}
53+
54+
async function instantiateWasmModule() {
55+
if (wasmBinary === undefined) {
56+
if (typeof Buffer !== 'undefined') { // node.js
57+
wasmBinary = new Uint8Array(Buffer.from(wasmBase64, 'base64'))
58+
} else if (typeof window !== 'undefined') { // browser
59+
const binaryString = atob(wasmBase64) // window.atob() is fast and safe for valid base64 strings
60+
wasmBinary = new Uint8Array(binaryString.length)
61+
for (let i = 0; i < binaryString.length; i++) {
62+
wasmBinary[i] = binaryString.charCodeAt(i)
63+
}
64+
}
65+
}
66+
return await wasmModule({wasmBinary})
67+
}
68+
69+
export async function getTolkCompilerVersion(): Promise<string> {
70+
const mod = await instantiateWasmModule()
71+
72+
const versionJsonPtr = mod._version()
73+
const result = JSON.parse(copyFromCString(mod, versionJsonPtr))
74+
mod._free(versionJsonPtr)
75+
76+
return result.tolkVersion
77+
}
78+
79+
export async function runTolkCompiler(compilerConfig: TolkCompilerConfig): Promise<TolkResultSuccess | TolkResultError> {
80+
const mod = await instantiateWasmModule()
81+
const allocatedPointers = []
82+
const sourcesSnapshot: TolkResultSuccess['sourcesSnapshot'] = []
83+
84+
// see tolk-wasm.cpp: typedef void (*WasmFsReadCallback)(int, char const*, char**, char**)
85+
const callbackPtr = mod.addFunction(function (kind: number, dataPtr: any, destContents: any, destError: any) {
86+
switch (kind) { // enum ReadCallback::Kind in C++
87+
case 0: // realpath
88+
const relativeFilename = copyFromCString(mod, dataPtr)
89+
allocatedPointers.push(copyToCStringPtr(mod, realpath(relativeFilename), destContents))
90+
break
91+
case 1: // read file
92+
try {
93+
const filename = copyFromCString(mod, dataPtr) // already normalized (as returned above)
94+
if (filename.startsWith('@stdlib/')) {
95+
const stdlibKey = filename.endsWith('.tolk') ? filename : filename + '.tolk'
96+
if (!(stdlibKey in stdlibContents)) {
97+
allocatedPointers.push(copyToCStringPtr(mod, stdlibKey + " not found", destError))
98+
} else {
99+
allocatedPointers.push(copyToCStringPtr(mod, stdlibContents[stdlibKey], destContents))
100+
}
101+
} else {
102+
const contents = compilerConfig.fsReadCallback(filename)
103+
sourcesSnapshot.push({ filename, contents })
104+
allocatedPointers.push(copyToCStringPtr(mod, contents, destContents))
105+
}
106+
} catch (err: any) {
107+
allocatedPointers.push(copyToCStringPtr(mod, err.message || err.toString(), destError))
108+
}
109+
break
110+
default:
111+
allocatedPointers.push(copyToCStringPtr(mod, 'Unknown callback kind=' + kind, destError))
112+
break
113+
}
114+
}, 'viiii')
115+
116+
const configStr = JSON.stringify({ // undefined fields won't be present, defaults will be used, see tolk-wasm.cpp
117+
entrypointFileName: compilerConfig.entrypointFileName,
118+
optimizationLevel: compilerConfig.optimizationLevel,
119+
withStackComments: compilerConfig.withStackComments,
120+
experimentalOptions: compilerConfig.experimentalOptions,
121+
})
122+
123+
const configStrPtr = copyToCStringAllocating(mod, configStr)
124+
allocatedPointers.push(configStrPtr)
125+
126+
const resultPtr = mod._tolk_compile(configStrPtr, callbackPtr)
127+
allocatedPointers.push(resultPtr)
128+
const result: TolkResultSuccess | TolkResultError = JSON.parse(copyFromCString(mod, resultPtr))
129+
130+
allocatedPointers.forEach(ptr => mod._free(ptr))
131+
mod.removeFunction(callbackPtr)
132+
133+
return result.status === 'error' ? result : {...result, sourcesSnapshot}
134+
}

src/path-utils.ts

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
const SLASH = 47
2+
const DOT = 46
3+
4+
// this function is directly from node source
5+
function posixNormalize(path: string, allowAboveRoot: boolean): string {
6+
let res = ''
7+
let lastSegmentLength = 0
8+
let lastSlash = -1
9+
let dots = 0
10+
let code
11+
12+
for (let i = 0; i <= path.length; ++i) {
13+
if (i < path.length) {
14+
code = path.charCodeAt(i)
15+
} else if (code === SLASH) {
16+
break
17+
} else {
18+
code = SLASH
19+
}
20+
if (code === SLASH) {
21+
if (lastSlash === i - 1 || dots === 1) {
22+
// NOOP
23+
} else if (lastSlash !== i - 1 && dots === 2) {
24+
if (
25+
res.length < 2 ||
26+
lastSegmentLength !== 2 ||
27+
res.charCodeAt(res.length - 1) !== DOT ||
28+
res.charCodeAt(res.length - 2) !== DOT
29+
) {
30+
if (res.length > 2) {
31+
const lastSlashIndex = res.lastIndexOf('/')
32+
if (lastSlashIndex !== res.length - 1) {
33+
if (lastSlashIndex === -1) {
34+
res = ''
35+
lastSegmentLength = 0
36+
} else {
37+
res = res.slice(0, lastSlashIndex)
38+
lastSegmentLength = res.length - 1 - res.lastIndexOf('/')
39+
}
40+
lastSlash = i
41+
dots = 0
42+
continue
43+
}
44+
} else if (res.length === 2 || res.length === 1) {
45+
res = ''
46+
lastSegmentLength = 0
47+
lastSlash = i
48+
dots = 0
49+
continue
50+
}
51+
}
52+
if (allowAboveRoot) {
53+
if (res.length > 0) {
54+
res += '/..'
55+
} else {
56+
res = '..'
57+
}
58+
lastSegmentLength = 2
59+
}
60+
} else {
61+
if (res.length > 0) {
62+
res += '/' + path.slice(lastSlash + 1, i)
63+
} else {
64+
res = path.slice(lastSlash + 1, i)
65+
}
66+
lastSegmentLength = i - lastSlash - 1
67+
}
68+
lastSlash = i
69+
dots = 0
70+
} else if (code === DOT && dots !== -1) {
71+
++dots
72+
} else {
73+
dots = -1
74+
}
75+
}
76+
77+
return res
78+
}
79+
80+
// 'realpath' in Tolk internals is used to resolve #include
81+
// (e.g., to detect that #include "a.tolk" and #include "dir/../a.tolk" reference the same file)
82+
// here we do the same using manual normalization, taken from Node internals (to work in web)
83+
//
84+
// also, 'realpath' in C++ resolves symlinks
85+
// here, in tolk-js, we don't do anything about it, since we don't perform actual file reading
86+
export function realpath(p: string): string {
87+
let isAbsolute = p.charCodeAt(0) === SLASH
88+
let path = posixNormalize(p, !isAbsolute)
89+
90+
if (isAbsolute) { // posixNormalize() drops leading slash
91+
return '/' + path
92+
}
93+
return path.length === 0 ? '.' : path
94+
}

0 commit comments

Comments
 (0)