🐛 with corpus build script for columns

Author: wolfram77

.github/workflows/ci.yml

@@ -41,4 +41,9 @@ jobs:
       id-token: write
     steps:
       - uses: actions/checkout@v4
-      - run: npx jsr publish
+      - uses: denoland/setup-deno@v2
+        with:
+          deno-version: v2.x
+      - run: npm install -g browserify uglify-js
+      - run: deno --allow-all build.ts ci
+      - run: npx jsr publish --allow-dirty

.gitignore

@@ -1,7 +1,8 @@
 # Generated files
 .build/
 *.map
-example.js
+corpus.js
+*.umd.js
 
 # Logs
 logs

about/build.ts

@@ -1,29 +1,31 @@
 // Copyright (C) 2025 Subhajit Sahu
 // SPDX-License-Identifier: AGPL-3.0-or-later
 // See LICENSE for full terms
-function readTextFiles(dir: string) {
+import * as path from "jsr:@std/path@1.0.9";
+
+
+async function readTextFiles(dir: string) {
   const a = new Map<string, string>();
   for (const f of Deno.readDirSync(dir)) {
     if (!f.isFile || !f.name.endsWith('.txt')) continue;
     const name    = f.name.replace('.txt', '');
-    const content = Deno.readTextFileSync(`assets/${f.name}`);
+    const content = await Deno.readTextFile(path.join(dir, f.name));
     a.set(name, content);
   }
   return a;
 }
 
 
-function writeIndex(file: string, data: Map<string, string>) {
+async function writeIndex(file: string, data: Map<string, string>) {
   let a = '';
   for (const [k, v] of data)
-    a += `[[${k}]]\n${v}\n\n\n`;
-  Deno.writeTextFileSync(file, a);
+    a += `[[${k}]]\n${v}\n\n`;
+  await Deno.writeTextFile(file, a.trim() + '\n');
   console.log(`Wrote ${data.size} entries to ${file}`);
 }
 
 
-function main() {
-  const data = readTextFiles('assets');
-  writeIndex('index.txt', data);
+export async function build() {
+  const data = await readTextFiles(path.join(import.meta.dirname || '', 'assets'));
+  await writeIndex(path.join(import.meta.dirname || '', 'index.txt'), data);
 }
-main();

build.ts

@@ -0,0 +1,36 @@
+import * as about   from "./about/build.ts";
+import * as columns from "./columns/build.ts";
+import * as columndescriptions from "./columndescriptions/build.ts";
+import * as hierarchy        from "./hierarchy/build.ts";
+import * as representations  from "./representations/build.ts";
+import * as compositions     from "./compositions/build.ts";
+import * as compositionstats from "./compositionstats/build.ts";
+
+
+async function main() {
+  await columns.build('./columns/corpus.js');
+  await columndescriptions.build('./columndescriptions/corpus.js');
+  await hierarchy.build('./hierarchy/corpus.js');
+  await representations.build('./representations/corpus.js');
+  if (Deno.args.length === 0) {
+    await about.build();
+    await compositions.build();
+    await compositionstats.build();
+  }
+  await Deno.writeTextFile('corpus.js',
+    `exports.columns   = require('./columns/corpus.js');\n` +
+    `exports.hierarchy = require('./hierarchy/corpus.js');\n` +
+    `exports.representations = require('./representations/corpus.js');\n`
+  );
+  await (new Deno.Command('browserify', {
+    args: ['corpus.js', '-s', 'ifct2017', '-o', 'corpus.umd.js'],
+    stdin: 'piped', stdout: 'piped', stderr: 'piped'
+  })).spawn().output();
+  await (new Deno.Command('uglifyjs', {
+    args: ['corpus.umd.js', '-c', '-m', '-o', 'corpus.min.js'],
+    stdin: 'piped', stdout: 'piped', stderr: 'piped'
+  })).spawn().output();
+  await Deno.remove('corpus.js');
+  await Deno.remove('corpus.umd.js');
+}
+main();

columndescriptions/build.ts

@@ -3,12 +3,20 @@ import * as csv  from "jsr:@std/csv@1.0.6";
 
 
 
+async function writeCorpus(outfile: string, records: Record<string, string>[]) {
+  let a = `var CORPUS = new Map([\n`;
+  for (const r of records)
+    a += `  ["${r.code}", ${JSON.stringify(r)}],\n`;
+  a += `]);\n`;
+  a += `module.exports = CORPUS;\n`;
+  await Deno.writeTextFile(outfile, a);
+}
+
 
-async function main() {
+export async function build(corpus='corpus.js') {
+  if (!corpus) return;
   const file = path.join(import.meta.dirname || "", "index.csv");
   const data = await Deno.readTextFile(file);
   const records  = csv.parse(data, {skipFirstRow: true, comment: "#"});
-  const jsonfile = path.join(import.meta.dirname || "", "index.json");
-  await Deno.writeTextFile(jsonfile, JSON.stringify(records, null, 2));
+  await writeCorpus(corpus, records);
 }
-main();

columns/build.ts

@@ -0,0 +1,23 @@
+import * as path from "jsr:@std/path@1.0.9";
+import * as csv  from "jsr:@std/csv@1.0.6";
+
+
+async function writeCorpus(outfile: string) {
+  const infile  = path.join(import.meta.dirname || "", "index.csv");
+  const data    = await Deno.readTextFile(infile);
+  const records = csv.parse(data, {skipFirstRow: true, comment: "#"});
+  const map     = new Map();
+  for (const r of records)
+    map.set(r.code, r);
+  let a = `var CORPUS = new Map([\n`;
+  for (const [k, v] of map)
+    a += `  ["${k}", ${JSON.stringify(v).replace(/\"(\w+)\":/g, '$1:')}],\n`;
+  a += `]);\n`;
+  a += `module.exports = CORPUS;\n`;
+  await Deno.writeTextFile(outfile, a);
+}
+
+
+export async function build(corpus='corpus.js') {
+  if (corpus) await writeCorpus(corpus);
+}

compositions/build.ts

@@ -3,7 +3,7 @@
 // See LICENSE for full terms
 import * as path from "jsr:@std/path@1.0.9";
 import * as csv  from "jsr:@std/csv@1.0.6";
-import {loadColumns, columns} from "../columns/index.ts";
+import {loadColumns} from "../columns/index.ts";
 import {type Description, loadDescriptions} from "../descriptions/index.ts";
 import {type Group, loadGroups} from "../groups/index.ts";
 
@@ -51,7 +51,7 @@ interface Dat {
 
 
 
-async function generateIndexCsv() {
+async function writeIndex() {
   let dat: Dat = {
     code: [],
     name: [],
@@ -167,21 +167,21 @@ async function generateIndexCsv() {
   }
 
 
-  async function main() {
+  async function writeIndexMain() {
     await loadColumns();
     grupCorpus = await loadGroups();
     descCorpus = await loadDescriptions();
-    factors = await readCsv('configs/factors.csv', (acc, r) => acc.set(r.code, r.factor),     new Map<string, string>());
-    renames = await readCsv('configs/renames.csv', (acc, r) => acc.set(r.code, r.actual),     new Map<string, string>());
-    sums    = await readCsv('configs/sums.csv',    (acc, r) => acc.set(r.code, r.expression), new Map<string, string>());
-    orders  = await readCsv('configs/orders.csv',  (acc, r) => {
+    factors = await readCsv(path.join(import.meta.dirname || '', 'configs/factors.csv'), (acc, r) => acc.set(r.code, r.factor),     new Map<string, string>());
+    renames = await readCsv(path.join(import.meta.dirname || '', 'configs/renames.csv'), (acc, r) => acc.set(r.code, r.actual),     new Map<string, string>());
+    sums    = await readCsv(path.join(import.meta.dirname || '', 'configs/sums.csv'),    (acc, r) => acc.set(r.code, r.expression), new Map<string, string>());
+    orders  = await readCsv(path.join(import.meta.dirname || '', 'configs/orders.csv'),  (acc, r) => {
       const arr = acc.get(r.before) || [];
       acc.set(r.before, arr);
       arr.push(r.code);
     }, new Map<string, string[]>());
-    for (const file of Deno.readDirSync('assets')) {
+    for (const file of Deno.readDirSync(path.join(import.meta.dirname || '', 'assets'))) {
       if (!file.name.endsWith('.csv')) continue;
-      await readAsset(path.join('assets', file.name));
+      await readAsset(path.join(import.meta.dirname || '', 'assets', file.name));
     }
     nullToZero(dat);
     sumAll(dat);
@@ -196,16 +196,12 @@ async function generateIndexCsv() {
       }
       a = a.substring(0, a.length-1) + '\n';
     }
-    Deno.writeTextFileSync('index.csv', a);
+    await Deno.writeTextFile(path.join(import.meta.dirname || '', 'index.csv'), a);
   }
-  await main();
+  await writeIndexMain();
 }
 
 
-
-
-// Finally.
-async function main() {
-  await generateIndexCsv();
+export async function build() {
+  await writeIndex();
 }
-main();

compositionstats/build.ts

@@ -61,7 +61,7 @@ function formatCompositionRow(row: Record<string, string | number>, code: string
 /** Get the minimum, maximum and average composition of all foods. */
 async function minMaxAvgComposition() {
   // Load the compositions of all foods.
-  const data = await Deno.readTextFile(path.join(import.meta.dirname || "", "index.csv"));
+  const data = await Deno.readTextFile(path.join(import.meta.dirname || "", "../compositions/index.csv"));
   const records: Record<string, string | number>[] = csv.parse(data, {skipFirstRow: true, comment: "#"});
   for (const row of records) {
     for (const k in row) {
@@ -88,12 +88,16 @@ async function minMaxAvgComposition() {
 
 
 /** Generate the stats CSV file. */
-async function main() {
+async function writeIndex() {
   const [min, max, avg] = await minMaxAvgComposition();
   const keys = Object.keys(min);
   const data = keys.join(",") + "\n" + [min, max, avg].map(row => {
     return keys.map(k => typeof row[k] === "number"? `${round(row[k] as number)}` : `"${row[k]}"`);
   }).join("\n") + "\n";
   await Deno.writeTextFile(path.join(import.meta.dirname || "", "index.csv"), data);
 }
-main();
+
+
+export async function build() {
+  await writeIndex();
+}