feat: automatic batching (#104)

* feat: evaluate multiple sequences in parallel with automatic batching
* feat: improve automatic chat wrapper resolution
* feat: smart context shifting
* feat: improve TS types
* refactor: improve API
* build: support beta releases
* build: improve dev configurations

BREAKING CHANGE: completely new API (docs will be updated before a stable version is released)

Author: giladgd

.config/typedoc.json

@@ -18,5 +18,6 @@
     "preserveAnchorCasing": true,
     "useCodeBlocks": true,
     "expandObjects": true,
-    "parametersFormat": "table"
+    "parametersFormat": "table",
+    "hideInPageTOC": true
 }

.editorconfig

@@ -10,8 +10,5 @@ insert_final_newline = true
 [{package.json,package-lock.json,manifest.json}]
 indent_size = 2
 
-[.babelrc]
-indent_size = 2
-
 [*.yml]
 indent_size = 2

.eslintrc.json

@@ -7,7 +7,8 @@
     },
     "ignorePatterns": ["/dist", "/llama", "/docs-site"],
     "extends": [
-        "eslint:recommended"
+        "eslint:recommended",
+        "plugin:jsdoc/recommended"
     ],
     "globals": {
         "Atomics": "readonly",
@@ -24,21 +25,31 @@
         "files": ["**.ts"],
         "extends": [
             "eslint:recommended",
-            "plugin:@typescript-eslint/recommended"
+            "plugin:@typescript-eslint/recommended",
+            "plugin:jsdoc/recommended-typescript"
         ],
         "parser": "@typescript-eslint/parser",
         "plugins": [
             "@typescript-eslint",
             "import",
-            "node"
+            "jsdoc",
+            "n"
         ],
         "rules": {
             "@typescript-eslint/explicit-module-boundary-types": ["off"],
             "@typescript-eslint/ban-ts-comment": ["off"],
             "@typescript-eslint/no-explicit-any": ["off"],
             "semi": ["off"],
             "@typescript-eslint/semi": ["warn", "always"],
-            "@typescript-eslint/no-inferrable-types": ["off"]
+            "@typescript-eslint/no-inferrable-types": ["off"],
+            "jsdoc/require-param": ["off"],
+            "jsdoc/check-param-names": ["warn", {
+                "checkDestructured": false
+            }],
+            "jsdoc/require-returns": ["off"],
+            "jsdoc/require-jsdoc": ["off"],
+            "jsdoc/require-yields": ["off"],
+            "jsdoc/require-param-description": ["off"]
         }
     }, {
         "files": ["test/**/**.ts"],
@@ -49,11 +60,18 @@
     "plugins": [
         "@typescript-eslint",
         "import",
-        "node"
+        "jsdoc",
+        "n"
     ],
     "settings": {
         "import/parsers": {
             "@typescript-eslint/parser": [".ts"]
+        },
+        "jsdoc": {
+            "exemptDestructuredRootsFromChecks": true,
+            "tagNamePreference": {
+                "hidden": "hidden"
+            }
         }
     },
     "rules": {
@@ -90,9 +108,7 @@
             "groups": ["builtin", "external","internal", "parent", "sibling", "index", "type", "object", "unknown"],
             "warnOnUnassignedImports": true
         }],
-        "node/file-extension-in-import": ["error", "always", {
-            "tryExtensions": [".js", ".json"]
-        }],
+        "n/file-extension-in-import": ["error", "always"],
         "newline-per-chained-call": ["error", {
             "ignoreChainWithDepth": 2
         }],
@@ -108,13 +124,6 @@
         }],
         "no-eval": ["error"],
         "array-callback-return": ["error"],
-        "valid-jsdoc": ["error", {
-            "requireParamType": true,
-            "requireReturnType": true,
-            "requireReturn": false,
-            "requireParamDescription": false,
-            "requireReturnDescription": false
-        }],
         "no-empty": ["error", {
             "allowEmptyCatch": true
         }],

.github/FUNDING.yml

@@ -1,6 +1,6 @@
 # These are supported funding model platforms
 
-github: [giladgd, ido-pluto]
+github: giladgd
 patreon: # Replace with a single Patreon username
 open_collective: # Replace with a single Open Collective username
 ko_fi: # Replace with a single Ko-fi username

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -68,7 +68,7 @@ body:
       description: >-
         Add any other context about the bug report here.
   - type: checkboxes
-    id: drivers
+    id: features
     attributes:
       label: Relevant Features Used
       options:

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -43,7 +43,7 @@ body:
       label: Additional Context
       description: Add any other context about the feature request here
   - type: checkboxes
-    id: drivers
+    id: features
     attributes:
       label: Related Features to This Feature Request
       options:

.github/workflows/build.yml

@@ -239,7 +239,7 @@ jobs:
 
   release:
     name: Release
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/beta'
     runs-on: ubuntu-latest
     concurrency: release-${{ github.ref }}
     environment:
@@ -296,20 +296,20 @@ jobs:
             echo "npm-url=https://www.npmjs.com/package/node-llama-cpp/v/$(cat .semanticRelease.npmPackage.deployedVersion.txt)" >> $GITHUB_OUTPUT
           fi
       - name: Generate docs with updated version
-        if: steps.set-npm-url.outputs.npm-url != ''
+        if: steps.set-npm-url.outputs.npm-url != '' && github.ref == 'refs/heads/master'
         env:
           DOCS_URL_BASE: "/node-llama-cpp/"
         run: |
           export DOCS_PACKAGE_VERSION=$(cat .semanticRelease.npmPackage.deployedVersion.txt)
           npm run docs:build
       - name: Upload docs to GitHub Pages
-        if: steps.set-npm-url.outputs.npm-url != ''
+        if: steps.set-npm-url.outputs.npm-url != '' && github.ref == 'refs/heads/master'
         uses: actions/upload-pages-artifact@v2
         with:
           name: pages-docs
           path: docs-site
       - name: Deploy docs to GitHub Pages
-        if: steps.set-npm-url.outputs.npm-url != ''
+        if: steps.set-npm-url.outputs.npm-url != '' && github.ref == 'refs/heads/master'
         uses: actions/deploy-pages@v2
         with:
           artifact_name: pages-docs

.releaserc.json

@@ -1,6 +1,7 @@
 {
     "branches": [
-        "master"
+        "master",
+        {"name": "beta", "prerelease": true}
     ],
     "ci": true,
     "plugins": [

.vitepress/config.ts

@@ -115,9 +115,17 @@ export default defineConfig({
                     text: "npm",
                     link: "https://www.npmjs.com/package/node-llama-cpp"
                 }, {
-                    text: "Contributing",
+                    text: "Contribute",
                     link: "/guide/contributing"
-                }]
+                },
+                ...(
+                    packageJson?.funding?.url == null
+                        ? []
+                        : [{
+                            text: "Sponsor",
+                            link: packageJson?.funding?.url
+                        }]
+                )]
             }
         ],
         search: {
@@ -258,29 +266,37 @@ function orderTypes(sidebar: typeof typedocSidebar) {
     if (types == null || !(types.items instanceof Array))
         return;
 
-    function groupGbnfJsonSchema() {
+    function groupItems(
+        findParent: (item: DefaultTheme.SidebarItem) => boolean | undefined,
+        findChildren: (item: DefaultTheme.SidebarItem) => boolean | undefined,
+        {collapsed = true, moveToEndIfGrouped = true}: {collapsed?: boolean, moveToEndIfGrouped?: boolean} = {}
+    ) {
+        const children: DefaultTheme.SidebarItem[] = [];
+
         if (types == null || !(types.items instanceof Array))
             return;
 
-        const gbnfJsonSchemaItemTitle = "GbnfJsonSchema";
-        const gbnfItemsPrefix = "GbnfJson";
-        const gbnfJsonSchemaItems: DefaultTheme.SidebarItem[] = [];
-
-        const gbnfJsonSchemaItem = types.items
-            .find((item) => item.text === gbnfJsonSchemaItemTitle) as DefaultTheme.SidebarItem | null;
+        const parent = types.items.find(findParent) as DefaultTheme.SidebarItem | null;
 
-        if (gbnfJsonSchemaItem == null)
+        if (parent == null)
             return;
 
-        gbnfJsonSchemaItem.collapsed = true;
-        gbnfJsonSchemaItem.items = gbnfJsonSchemaItems;
-
         for (const item of types.items.slice()) {
-            if (item.text === gbnfJsonSchemaItemTitle || !item.text.startsWith(gbnfItemsPrefix))
+            if (item === parent || !findChildren(item))
                 continue;
 
             types.items.splice(types.items.indexOf(item), 1);
-            gbnfJsonSchemaItems.push(item);
+            children.push(item);
+        }
+
+        if (children.length > 0) {
+            parent.collapsed = collapsed;
+            parent.items = children;
+
+            if (moveToEndIfGrouped) {
+                types.items.splice(types.items.indexOf(parent as typeof types.items[number]), 1);
+                types.items.push(parent as typeof types.items[number]);
+            }
         }
     }
 
@@ -298,6 +314,24 @@ function orderTypes(sidebar: typeof typedocSidebar) {
         });
     }
 
-    groupGbnfJsonSchema();
+    groupItems(
+        (item) => item.text === "BatchingOptions",
+        (item) => (
+            item.text === "BatchItem" ||
+            item.text === "CustomBatchingDispatchSchedule" ||
+            item.text === "CustomBatchingPrioritizeStrategy" ||
+            item.text === "PrioritizedBatchItem"
+        ),
+        {collapsed: false}
+    );
+    groupItems(
+        (item) => item.text === "LlamaContextOptions",
+        (item) => item.text === "BatchingOptions"
+    );
+    groupItems(
+        (item) => item.text === "GbnfJsonSchema",
+        (item) => item.text?.startsWith("GbnfJson")
+    );
+    
     moveCollapseItemsToTheEnd();
 }

README.md

@@ -10,7 +10,7 @@
 
 [![Build](https://github.com/withcatai/node-llama-cpp/actions/workflows/build.yml/badge.svg)](https://github.com/withcatai/node-llama-cpp/actions/workflows/build.yml)
 [![License](https://badgen.net/badge/color/MIT/green?label=license)](https://www.npmjs.com/package/node-llama-cpp)
-[![License](https://badgen.net/badge/color/TypeScript/blue?label=types)](https://www.npmjs.com/package/node-llama-cpp)
+[![Types](https://badgen.net/badge/color/TypeScript/blue?label=types)](https://www.npmjs.com/package/node-llama-cpp)
 [![Version](https://badgen.net/npm/v/node-llama-cpp)](https://www.npmjs.com/package/node-llama-cpp)
 
 </div>