[function scope extraction] include lexical scope as code

drinking-code · drinking-code · commit 6f950210a486 · 2023-03-30T19:58:21.000+02:00
diff --git a/bun.lockb b/bun.lockb
diff --git a/package.json b/package.json
@@ -47,6 +47,7 @@
   },
   "dependencies": {
     "@babel/core": "^7.21.0",
+    "@babel/generator": "^7.21.3",
     "@babel/parser": "^7.19.3",
     "@babel/traverse": "^7.21.2",
     "autoprefixer": "^10.4.12",
diff --git a/src/compiler/client-script/extract-function.ts b/src/compiler/client-script/extract-function.ts
@@ -1,7 +1,7 @@
 import fs from 'fs'
 
 import Parser from '../parser'
-import {traverseFast, FunctionExpression, ArrowFunctionExpression} from '@babel/types'
+import {traverseFast, FunctionExpression, ArrowFunctionExpression, Statement} from '@babel/types'
 import {HashType} from '../../jsx/VirtualElement'
 import {Scope} from './scope'
 import {getCurrentComponentHash} from '../template/template-builder'
@@ -16,7 +16,12 @@ const transpiler = new Bun.Transpiler({
 })
 
 const scopes = {}
-const lexicalScopes: { [id: HashType]: Array<Node | { importName: string, localName: string, filePath: string }> } = {}
+export type ImportDataType = { importName: string | typeof Scope.importAll | typeof Scope.defaultImport, localName: string, filePath: string }
+const lexicalScopes: { [id: HashType]: Array<Statement | ImportDataType>[] } = {}
+
+export function getLexicalScope(id: HashType, index: number) {
+    return lexicalScopes[id][index]
+}
 
 export async function extractFunction(
     {functionName, filePath, line, column}: { functionName: string, filePath: string, line: number, column: number }
@@ -27,21 +32,21 @@ export async function extractFunction(
     if (!parser.fileNames.includes(filePath)) {
         const fileContents = fs.readFileSync(filePath, {encoding: 'utf8'})
         const transformedFileContents = transpiler.transformSync(fileContents, 'tsx')
-        console.log(transformedFileContents.split("\n")[line - 1])
-        console.log(Array(column - 1).fill(' ').join('') + '^')
         parser.parseFile(filePath, transformedFileContents)
     }
 
-    const scope = scopes[filePath] ?? new Scope()
+    const scope: Scope = scopes[filePath] ?? new Scope()
     const thingsInUse = []
     parser.traverseFileFast(filePath, (node) => {
         if (!(filePath in scope)) {
             // declarations
             if (node.type === 'FunctionDeclaration') {
                 scope.add(node.id, node)
-            } else if (node.type === 'VariableDeclarator') {
-                if (node.id.type === 'Identifier')
-                    scope.add(node.id, node)
+            } else if (node.type === 'VariableDeclaration') {
+                node.declarations.forEach(declaration => {
+                    if (declaration.id.type === 'Identifier')
+                        scope.add(declaration.id, node)
+                })
             } else if (node.type === 'ImportDeclaration') {
                 node.specifiers.forEach(importSpecifier => {
                     let importName: string | typeof Scope.importAll | typeof Scope.defaultImport
@@ -89,18 +94,24 @@ export async function extractFunction(
             }
         })
     })
-    lexicalScopes[getCurrentComponentHash()] = thingsInUse
-        .map(thing => {
-            if (scope.has(scope.getId(thing))) {
-                return [
-                    scope.getOrder(scope.getId(thing)),
-                    scope.get(scope.getId(thing))
-                ]
-            } else if (scope.hasImport(thing)) {
-                return [0, scope.getImport(thing)]
-            }
-        })
-        .sort((a, b) => a[0] - b[0])
-        .map(a => a[1])
+    const id = getCurrentComponentHash()
+    if (!lexicalScopes[id])
+        lexicalScopes[id] = []
+    lexicalScopes[id].push(
+        thingsInUse
+            .map((thing): [number, Statement[] | ImportDataType] => {
+                if (scope.has(scope.getId(thing))) {
+                    return [
+                        scope.getOrder(scope.getId(thing)),
+                        scope.get(scope.getId(thing))
+                    ]
+                } else if (scope.hasImport(thing)) {
+                    return [0, scope.getImport(thing)]
+                }
+            })
+            .sort((a, b) => a[0] - b[0])
+            .map(<V>(a: [number, V]): V => a[1])
+            .flat()
+    )
     scopes[filePath] = scope
 }
diff --git a/src/compiler/client-script/generate-data-files.ts b/src/compiler/client-script/generate-data-files.ts
@@ -5,27 +5,28 @@ import {Volume} from 'memfs/lib/volume'
 
 import projectRoot, {resolve as resolveProjectRoot} from '../../utils/project-root'
 import {resolve as resolveModuleRoot} from '../../utils/module-root'
-import {mapObjectToArray} from '../../utils/iterate-object'
-import {randomBytes} from 'crypto'
-import {replaceAsync} from '../../utils/replace-async'
-import {addMarker, addRange} from '../profiler'
-import {getRefs, getStateFromPlaceholderId, stateIdPlaceholderPrefix} from '../states-collector'
+import {iterateObject} from '../../utils/iterate-object'
+import {addMarker} from '../profiler'
+import {getRefs} from '../states-collector'
 import {getAssetsFilePaths} from '../assets'
 import {clientTemplatesToJs, refsToJs} from '../generate-code'
 import {getStateUsagesAsCode} from '../template/state-usage'
 import {getStateListenersAsCode} from '../../state/do-something'
 
 const entryPoint = process.env.CHERRY_COLA_ENTRY
-const entryDir = path.dirname(entryPoint)
+export const entryDir = path.dirname(entryPoint)
 const mountFromSrc = ['runtime', 'messages', 'utils']
 export const outputPath = '/dist'
 export const virtualFilesPath = '/_virtual-files'
-export const stateListenersFilePath = path.join(virtualFilesPath, 'state-listeners.js')
+export const stateListenersFileName = 'state-listeners.js'
+export const stateListenersFilePath = path.join(virtualFilesPath, stateListenersFileName)
 export const refsAndTemplatesFilePath = path.join(virtualFilesPath, 'refs-and-templates.js')
 
+export const hfsEntryDir = entryDir.replace(projectRoot, '')
+
 const hfs: Volume = createHybridFs([
     [resolveProjectRoot('node_modules'), '/node_modules'],
-    [entryDir, entryDir.replace(projectRoot, '')],
+    [entryDir, hfsEntryDir],
     ...mountFromSrc.map(dir => [resolveModuleRoot('src', dir), '/' + dir]),
 ])
 
@@ -42,7 +43,14 @@ export async function generateClientScriptFile() {
     let inputFile = ''
     inputFile += getAssetsFilePaths().map(path => `import '${path}'`).join(newLine)
     inputFile += newLine
-    inputFile += getStateListenersAsCode()
+    iterateObject(getStateListenersAsCode(), ([fileName, fileContents]) => {
+        if (fileName === stateListenersFileName) {
+            inputFile += fileContents
+        } else {
+            const filePath = path.join(virtualFilesPath, fileName)
+            hfs.writeFileSync(filePath, fileContents)
+        }
+    })
     // inputFile += clientScripts.map(virtualPath => `import '${virtualPath}'`).join(newLine)
     hfs.writeFileSync(stateListenersFilePath, inputFile)
     addMarker('bundler', 'client-script-file')
diff --git a/src/compiler/client-script/scope.ts b/src/compiler/client-script/scope.ts
@@ -1,12 +1,13 @@
-import {Identifier} from '@babel/types'
+import {Identifier, Statement} from '@babel/types'
+import {ImportDataType} from './extract-function'
 
 export class Scope {
-    data: Map<Identifier, Array<Node>> = new Map()
-    imports: { [filePath: string]: { [localName: string]: string } } = {}
+    data: Map<Identifier, Array<Statement>> = new Map()
+    imports: { [filePath: string]: { [localName: string]: string | typeof Scope.importAll | typeof Scope.defaultImport } } = {}
     order: [Identifier, number][] = []
 
     static idMatch(idA: Identifier, idB: Identifier) {
-        return idA.name === idB.name
+        return idA?.name === idB?.name
     }
 
     static defaultImport = Symbol.for('defaultImport')
@@ -33,7 +34,7 @@ export class Scope {
         return false
     }
 
-    add(id: Identifier, value) {
+    add(id: Identifier, value: Statement) {
         this.ensureKey(id)
         this.order.push([id, this.data.get(id).length])
         this.data.get(id).push(value)
@@ -42,7 +43,7 @@ export class Scope {
     addImport(localName: string, importName: string | typeof Scope.importAll | typeof Scope.defaultImport, fileName) {
         if (!this.imports[fileName])
             this.imports[fileName] = {}
-        this.imports[fileName][localName] = localName
+        this.imports[fileName][localName] = importName
     }
 
     get(id: Identifier) {
@@ -61,8 +62,8 @@ export class Scope {
         return index * 1e3 + this.order[index][1]
     }
 
-    getImport(localName: string) {
-        for (const filePath in Object.values(this.imports)) {
+    getImport(localName: string): ImportDataType {
+        for (const filePath of Object.keys(this.imports)) {
             if (localName in this.imports[filePath])
                 return {
                     importName: this.imports[filePath][localName],
diff --git a/src/compiler/client-script/stringify-scope.ts b/src/compiler/client-script/stringify-scope.ts
@@ -0,0 +1,28 @@
+import {HashType} from '../../jsx/VirtualElement'
+import {getLexicalScope, ImportDataType} from './extract-function'
+import generate from '@babel/generator'
+import {program, Statement} from '@babel/types'
+import {Scope} from './scope'
+
+export function getStringifiedLexicalScope(id: HashType, index: number, transformImportPath: (path: string) => string) {
+    const lexicalScope = getLexicalScope(id, index)
+    const lexicalScopeStatements = lexicalScope.filter((element): element is Statement => 'kind' in element)
+    const lexicalScopeImports = lexicalScope.filter((element): element is ImportDataType => !('kind' in element))
+    const imports = lexicalScopeImports
+        .map(importData => {
+            let importNames
+            if (importData.importName === importData.localName) {
+                importNames = `{${importData.importName}}`
+            } else if (importData.importName === Scope.defaultImport) {
+                importNames = importData.localName
+            } else if (importData.importName === Scope.importAll) {
+                importNames = `* as ${importData.localName}`
+            } else {
+                importNames = `{${importData.importName as string} as ${importData.localName}}`
+            }
+            return `import ${importNames} from '${transformImportPath(importData.filePath)}'`
+        })
+        .join("\n")
+    const statements = generate(program(lexicalScopeStatements)).code
+    return imports + "\n" + statements
+}
diff --git a/src/runtime/state-listener.ts b/src/runtime/state-listener.ts
@@ -15,7 +15,7 @@ function prepStatesAndRefs(statesAndRefs: (State | Ref<any>)[]): ([any, (value:
 }
 
 export function registerStateListeners() {
-    // todo only execute currently used state listeners
+    // todo only execute state listeners of currently used components
     stateListeners.forEach((listeners, id) => {
         const parametersArray = stateListenersParameters.get(id)
         listeners.forEach((listener, index) => {
diff --git a/src/state/do-something.ts b/src/state/do-something.ts
@@ -7,6 +7,9 @@ import {getCurrentComponentHash} from '../compiler/template/template-builder'
 import {findNode} from '../runtime'
 import {getCallerPosition} from '../utils/get-caller-position'
 import {extractFunction} from '../compiler/client-script/extract-function'
+import {getStringifiedLexicalScope} from '../compiler/client-script/stringify-scope'
+import {entryDir, hfsEntryDir, stateListenersFileName} from '../compiler/client-script/generate-data-files'
+import path from 'path'
 
 export type StateOrRefType = State | Ref
 type StateType = State
@@ -41,7 +44,6 @@ const stateListeners: Map<HashType, StateListenerType<any>[]> = new Map()
 const stateListenersParameters: Map<HashType, StateOrRefType[][]> = new Map()
 
 function includeStateListener(callback: StateListenerType<any>, statesAndRefs: StateOrRefType[]) {
-    // todo: extract lexical scope of function
     const id = getCurrentComponentHash()
     if (!stateListeners.has(id))
         stateListeners.set(id, [])
@@ -51,44 +53,59 @@ function includeStateListener(callback: StateListenerType<any>, statesAndRefs: S
     stateListenersParameters.get(id).push(statesAndRefs)
 }
 
-export function getStateListenersAsCode() {
+export function getStateListenersAsCode(): { [fileName: string]: string } {
     const stateListenersName = 'stateListeners'
     const stateListenersParametersName = 'stateListenersParameters'
-    let code = ''
+    let mainFile = ''
     const newLine = "\n"
-    code += `import {${getClientState.name}} from '/runtime/client-state';` + newLine
-    code += `import {${findNode.name}} from '/runtime/dom';` + newLine
-    code += `const ${stateListenersName} = new Map();` + newLine
-    code += `const ${stateListenersParametersName} = new Map();` + newLine
+    mainFile += `import {${getClientState.name}} from '/runtime/client-state';` + newLine
+    mainFile += `import {${findNode.name}} from '/runtime/dom';` + newLine
+    mainFile += `const ${stateListenersName} = new Map();` + newLine
+    mainFile += `const ${stateListenersParametersName} = new Map();` + newLine
+    const files: { [fileName: string]: string } = {}
     stateListeners.forEach((callbacks, id) => {
+        const componentFileName = `${id}.js`
+        let componentFile = ''
         const statesAndRefArrays = stateListenersParameters.get(id)
-        code += '{' + newLine
-        // todo: put lexical stuff here
         let stringifiedCallbacks = '['
-        for (const callback of callbacks) {
-            stringifiedCallbacks += callback.toString()
+        for (const index in callbacks) {
+            // todo: put lexical stuff here
+            componentFile += getStringifiedLexicalScope(
+                id, Number(index),
+                importPath => path.join(entryDir, importPath)
+            ) + newLine
+            stringifiedCallbacks += callbacks[index].toString()
             stringifiedCallbacks += ','
         }
         stringifiedCallbacks += ']'
-        code += `${stateListenersName}.set('${id}', ${stringifiedCallbacks});` + newLine
-        let stateArrays = '['
+        componentFile += 'export const callbacks = ' + stringifiedCallbacks + newLine
+        mainFile += `import {callbacks as callbacks_${id}} from './${componentFileName}'` + newLine
+        mainFile += `${stateListenersName}.set('${id}', callbacks_${id});` + newLine
+        componentFile += 'export const states = ['
         for (const statesAndRefs of statesAndRefArrays) {
-            stateArrays += '['
+            componentFile += '['
             for (const stateOrRef of statesAndRefs) {
                 if (isRef(stateOrRef)) {
-                    stateArrays += `${findNode.name}('${stateOrRef.id}')`
+                    componentFile += `${findNode.name}('${stateOrRef.id}')`
                 } else {
-                    stateArrays += `${getClientState.name}('${stateOrRef.id}', ${JSON.stringify(stateOrRef.valueOf())})` // todo: use value from ast
+                    componentFile += `${getClientState.name}('${stateOrRef.id}', ${JSON.stringify(stateOrRef.valueOf())})` // todo: use value from ast
                 }
-                stateArrays += ', '
+                componentFile += ', '
             }
-            stateArrays += '],'
+            componentFile += '],'
         }
-        stateArrays += ']'
-        code += `${stateListenersParametersName}.set('${id}', ${stateArrays});` + newLine
-        code += '}' + newLine
+        componentFile += '];'
+        mainFile += `import {states as states_${id}} from './${componentFileName}'` + newLine
+        mainFile += `${stateListenersParametersName}.set('${id}', states_${id});` + newLine
+        files[componentFileName] = componentFile
+        // console.log()
+        // console.log(componentFile)
     })
-    code += `export {${stateListenersName}};` + newLine
-    code += `export {${stateListenersParametersName}};`
-    return code
+    mainFile += `export {${stateListenersName}};` + newLine
+    mainFile += `export {${stateListenersParametersName}};`
+    // return code
+    files[stateListenersFileName] = mainFile
+    // console.log()
+    // console.log(mainFile)
+    return files
 }

Original file line number	Diff line number	Diff line change
`@@ -15,7 +15,7 @@ function prepStatesAndRefs(statesAndRefs: (State \| Ref<any>)[]): ([any, (value:`
`15`	`15`	`}`
`16`	`16`
`17`	`17`	`export function registerStateListeners() {`
`18`		`- // todo only execute currently used state listeners`
	`18`	`+ // todo only execute state listeners of currently used components`
`19`	`19`	`stateListeners.forEach((listeners, id) => {`
`20`	`20`	`const parametersArray = stateListenersParameters.get(id)`
`21`	`21`	`listeners.forEach((listener, index) => {`