[Fix][Grammar] Detach token table to prevent disposing it (#571)

Prior to this PR, when reusing the same engine but for different schema
will run into error "Module has already been disposed". An example to
reproduce this is included in
#560.

This is because `this.tokenTable` is a `tvmjs.TVMObject` and will be
disposed after the scope ends.

We fix this by wrapping the call with
`this.tvm.detachFromCurrentScope()`, and only dispose `this.tokenTable`
when we dispose the entire `LLMChatPipeline`.

Co-authored-by: SMarioMan <[email protected]>

Author: CharlieFRuan

src/llm_chat.ts

@@ -298,6 +298,7 @@ export class LLMChatPipeline {
     this.logitsOnCPU?.dispose();
     this.tvm.dispose();
     this.tokenizer.dispose();
+    this.tokenTable?.dispose();
   }
 
   /**
@@ -557,9 +558,11 @@ export class LLMChatPipeline {
         if (this.tokenTable === undefined) {
           const rawTokenTable = getTokenTableFromTokenizer(this.tokenizer);
           // Post process entire table
-          this.tokenTable = this.fpostProcessTokenTable(
-            rawTokenTable,
-            this.token_postproc_method,
+          this.tokenTable = this.tvm.detachFromCurrentScope(
+            this.fpostProcessTokenTable(
+              rawTokenTable,
+              this.token_postproc_method,
+            ),
           );
         }
         const grammar: BNFGrammar =