move to token-based parsing

Author: thomasjball

fieldeditors.ts

@@ -35,7 +35,7 @@ namespace microcode {
         toString(field: any): string {
             return ""
         }
-        fromString(text: string): any {
+        fromTokens(tokens: string[]): any {
             return undefined
         }
     }
@@ -102,8 +102,9 @@ namespace microcode {
         toString(field: BoxedNumAsStr) {
             return field.num
         }
-        fromString(txt: string) {
-            return { num: txt }
+        fromTokens(tokens: string[]) {
+            // TODO: check that we have a number
+            return { num: tokens.length > 0 ? tokens[0] : "0" }
         }
     }
 
@@ -183,22 +184,21 @@ namespace microcode {
             return img
         }
         toString(img: Bitmap) {
-            let ret = ""
+            let ret = "`"
             for (let row = 0; row < 5; row++) {
                 for (let col = 0; col < 5; col++) {
                     ret += img.getPixel(col, row) ? `1` : `.`
                     if (col < 4) ret += " "
                 }
                 ret += `\n`
             }
-            return ret
+            return ret + "`"
         }
-        fromString(txt: string): Bitmap {
+        fromTokens(tokens: string[]): Bitmap {
             let ret = bitmaps.create(5, 5)
-            let seq = txt.split("\n").join(" ").split(" ")
-            seq.forEach((s, i) => {
-                ret.setPixel(i % 5, Math.idiv(i, 5), s == "1" ? 1 : 0)
-            })
+            for (let i = 0; i < tokens.length && i < 25; i++) {
+                ret.setPixel(i % 5, Math.idiv(i, 5), tokens[i] == "1" ? 1 : 0)
+            }
             return ret
         }
     }
@@ -235,20 +235,22 @@ namespace microcode {
 
     export function melodyToNotes(melody: Melody) {
         const notes = melody.notes.split("")
-        let result = ""
+        let result = "`"
         for (const n of notes) {
             if (n == ".") result += "- "
             else result += noteNames[parseInt(n)] + " "
         }
-        return result
+        return result + "`"
     }
 
-    function notesToMelody(txt: string) {
-        const seq = txt.split(" ")
+    function notesToMelody(tokens: string[]) {
         let res = ""
-        seq.forEach((note, index) => {
+        tokens.forEach((note, index) => {
             if (note == "-") res += "."
-            else res += noteNames.indexOf(note).toString()
+            else {
+                const index = noteNames.indexOf(note)
+                if (index >= 0) res += index.toString()
+            }
         })
         return { notes: res, tempo: 120 }
     }
@@ -312,8 +314,8 @@ namespace microcode {
         toString(melody: Melody): string {
             return melodyToNotes(melody)
         }
-        fromString(txt: string): Melody {
-            return notesToMelody(txt)
+        fromTokens(tokens: string[]): Melody {
+            return notesToMelody(tokens)
         }
     }
 

mcparser.ts

@@ -56,71 +56,74 @@ namespace microcode {
             }
         }
         const addTile = (tile: Tile, rule: RuleDefn) => {
+            control.assert(rule != undefined)
             const tid = getTid(tile)
+            if (isSensor(tid)) rule.push(tile, "sensors", false)
             if (isFilter(tid)) rule.push(tile, "filters", false)
             else if (isModifier(tid)) rule.push(tile, "modifiers", false)
             else rule.push(tile, "actuators", false)
         }
+        let cursor = 0
+        const whiteSpace = (s: string) => {
+            return s == " " || s == "\n" || s == "\t"
+        }
+        const getToken = () => {
+            let prev = cursor
+            let gotToken = false
+            while (cursor < str.length) {
+                if (whiteSpace(str[cursor])) {
+                    if (gotToken) return str.slice(prev, cursor)
+                    cursor++
+                    prev = cursor
+                } else {
+                    gotToken = true
+                    cursor++
+                }
+            }
+            if (gotToken) return str.slice(prev, cursor)
+            return undefined
+        }
         const prog = new ProgramDefn()
         prog.pages = []
-        const lines = str.split("\n")
+
         let currPage: PageDefn = undefined
         let currRule: RuleDefn = undefined
         let currTile: Tile = undefined
-        for (let i = 0; i < lines.length; i++) {
-            console.log(`lines[${i}] = ${lines[i]}`)
-            if (currTile && currTile instanceof IconEditor) {
-                const all5 = lines.slice(i, i + 5).join("\n")
-                console.log(`get the image?\n${all5}`)
-                currTile.field = currTile.fieldEditor.fromString(all5)
-                currTile = undefined
-                i = i + 4 // loop count adds one more
-                continue
-            }
-            const tokens = lines[i].split(" ")
-            if (tokens.length == 0) continue
-            console.log(`tokens = ${tokens.join(":")}`)
-            let tok = tokens.shift()
+        let tok: string = undefined
+        while ((tok = getToken())) {
             console.log(`tok1 = ${tok}`)
-            if (tok == "Page") {
+            if (currTile) {
+                if (
+                    currTile instanceof IconEditor ||
+                    currTile instanceof MelodyEditor
+                ) {
+                    control.assert(tok == "`")
+                    let iconTokens = []
+                    while ((tok = getToken()) != "`") iconTokens.push(tok)
+                    control.assert(tok == "`")
+                    currTile.field = currTile.fieldEditor.fromTokens(iconTokens)
+                } else if (currTile instanceof DigitEditor) {
+                    currTile.field = currTile.fieldEditor.fromTokens([tok])
+                }
+                currTile = undefined
+            } else if (tok == "Page") {
                 if (currPage) {
                     if (currRule) currPage.rules.push(currRule)
                     prog.pages.push(currPage)
                     currRule = undefined
                 }
                 currPage = new PageDefn()
+                getToken() // consume page #
                 continue
             } else if (tok == "When") {
                 control.assert(currPage != undefined)
                 if (currRule) currPage.rules.push(currRule)
-                currRule = undefined
-                tok = tokens.shift()
-            }
-            for (; tok !== undefined; tok = tokens.shift()) {
-                console.log(`tok2 = ${tok}`)
-                if (!tok) continue
-                if (!currRule) {
-                    currRule = new RuleDefn()
-                    // can we have When followed by Do?
-                    currRule.sensors.push(token2tile(tok) as number)
-                    continue
-                }
-                if (tok == "Do") continue
+                currRule = new RuleDefn()
+            } else if (tok == "Do") {
+                control.assert(currRule != undefined)
+            } else {
                 currTile = token2tile(tok)
                 addTile(currTile, currRule)
-                if (currTile instanceof IconEditor) {
-                    console.log(`got IconEditor`)
-                    break
-                } else if (currTile instanceof DigitEditor) {
-                    currTile.field = currTile.fieldEditor.fromString(tok)
-                    currTile = undefined
-                } else if (currTile instanceof MelodyEditor) {
-                    currTile.field = currTile.fieldEditor.fromString(
-                        tok + tokens.join(" ")
-                    )
-                    currTile = undefined
-                    break
-                }
             }
         }
         if (currRule) currPage.rules.push(currRule)