Generate notes from in-source file links (#155)

* Add suggestion to fix the links.

* Generate notes from links.

* Re-export notev3.

Author: tomusdrw

shared/links-metadata/index.ts

@@ -6,3 +6,4 @@ export { type Link, parseAndMigrateLink, parseLink, findLink } from "./src/link"
 export * from "./src/metadata";
 export * from "./src/types";
 export * from "./src/utils";
+export * from "./src/notes";

shared/links-metadata/src/link.ts

@@ -2,7 +2,7 @@ import type { SynctexStore } from "./SynctexStore";
 import type { TexStore } from "./TexStore";
 import { type Metadata, ORIGIN } from "./metadata";
 import { migrateSelection } from "./migrate";
-import type { ISynctexBlockId } from "./types";
+import type { ISelectionParams, ISynctexBlockId } from "./types";
 
 export type Link = {
   url: string;
@@ -12,7 +12,7 @@ export type Link = {
   version: string;
   versionName: string;
   isOutdated: boolean;
-};
+} & ISelectionParams;
 
 export function findLink(line: string) {
   const linkStart = line.indexOf(ORIGIN);
@@ -73,6 +73,8 @@ export async function parseAndMigrateLink(
       versionName: "legacy",
       isOutdated: true,
       migrated: false,
+      selectionStart: { pageNumber: 0, index: 0 },
+      selectionEnd: { pageNumber: 0, index: 0 },
     };
   }
   const { version, versionName, selectionStart, selectionEnd } = linkData;
@@ -112,6 +114,8 @@ export async function parseAndMigrateLink(
     versionName,
     isOutdated,
     migrated,
+    selectionStart,
+    selectionEnd,
   };
 }
 

shared/links-metadata/src/notes.ts

@@ -0,0 +1,22 @@
+import type { ISelectionParams } from "./types";
+
+/** Notes envelope. */
+export interface INotesEnvelopeV3 {
+  version: 3;
+  notes: INoteV3[];
+}
+
+/** Version 3 of the note format. */
+export interface INoteV3 extends ISelectionParams {
+  /** Constant value for this version of the note. */
+  noteVersion: 3;
+  content: string;
+  /** local-tz timestamp (from Date.now()) */
+  date: number;
+  /** empty for local notes. */
+  author: string;
+  /** Full version number. */
+  version: string;
+  /** Labels. */
+  labels: string[];
+}

src/components/NotesProvider/types/StorageNote.ts

@@ -1,20 +1,8 @@
-import type { ISelectionParams } from "@fluffylabs/links-metadata";
+import type { INoteV3, INotesEnvelopeV3 } from "@fluffylabs/links-metadata";
+export type { INoteV3 } from "@fluffylabs/links-metadata";
 
 /** Latest version of the stored note. */
 export type IStorageNote = INoteV3;
 
-/** Notes envelope. */
-export interface INotesEnvelope {
-  version: 3;
-  notes: INoteV3[];
-}
-
-/** Version 3 of the note format. */
-export interface INoteV3 extends ISelectionParams {
-  noteVersion: 3;
-  content: string;
-  date: number;
-  author: string;
-  version: string;
-  labels: string[];
-}
+/** Versioned wrapper for notes. */
+export type INotesEnvelope = INotesEnvelopeV3;

tools/links-check/index.ts

@@ -5,6 +5,7 @@ import { program } from "commander";
 import fastGlob from "fast-glob";
 import ignore from "ignore";
 import { performMigrations } from "./migrate";
+import { generateNotes } from "./notes";
 import { type Report, printReport } from "./report";
 import { scan } from "./scan";
 
@@ -21,8 +22,9 @@ async function main() {
       "--ignore-file <path>",
       "Path to a file containing patterns to ignore. Gitignore format applies. Patterns are resolved according to current working directory.",
     )
-    .option("--write", "Perform link migrations in provided source files.")
+    .option("--write", "Modify the files and update reader links to their newest versions.")
     .option("--fix", "Alias for --write.")
+    .option("--generate-notes <file.json>", "Generate notes for the Gray Paper Reader")
     .action(async (paths, options) => {
       let files = await fastGlob(paths);
       const globExpandedFileCount = files.length;
@@ -73,6 +75,13 @@ async function main() {
         return;
       }
 
+      if (options.generateNotes !== undefined) {
+        console.info(`📓 Generating notes to ${options.generateNotes}`);
+        const notes = generateNotes(report);
+        const notesStr = JSON.stringify(notes, null, 2);
+        fs.writeFileSync(options.generateNotes, notesStr);
+      }
+
       const summary = printReport(report);
 
       if (options.write || options.fix) {

tools/links-check/notes.ts

@@ -0,0 +1,66 @@
+import type { INoteV3, INotesEnvelopeV3, Link } from "@fluffylabs/links-metadata";
+import type { Report } from "./report";
+
+export function generateNotes(report: Report): INotesEnvelopeV3 {
+  const perLink = new Map<string, { file: string; link: Link }[]>();
+  const commonPath: Map<string, number>[] = [];
+
+  for (const [file, links] of report.detected) {
+    const parts = file.split("/");
+    for (let i = 0; i < parts.length; i++) {
+      commonPath[i] = commonPath[i] || new Map();
+      commonPath[i].set(parts[i], (commonPath[i].get(parts[i]) || 0) + 1);
+    }
+
+    for (const link of links) {
+      // consolidate urls to the newest version
+      const url = link.updated || link.url;
+      const list = perLink.get(url) ?? [];
+      list.push({
+        file,
+        link,
+      });
+      perLink.set(url, list);
+    }
+  }
+
+  const noOfFiles = report.detected.size;
+  const path = commonPath
+    .map((x) => {
+      for (const [k, count] of x.entries()) {
+        if (count === noOfFiles) {
+          return k;
+        }
+        return null;
+      }
+    })
+    .filter((x) => x)
+    .join("/");
+
+  const notes = [];
+  for (const [_, linkData] of perLink) {
+    notes.push(linkToNote(linkData, path));
+  }
+
+  return {
+    version: 3,
+    notes,
+  };
+}
+
+function linkToNote(linkData: { file: string; link: Link }[], path: string): INoteV3 {
+  const { link } = linkData[0];
+  // if we knew the github repository address this could actually be links to browse the files.
+  const content = linkData.map((x) => `${x.file.replace(path, "")}:${x.link.lineNumber}`).join("\n");
+
+  return {
+    noteVersion: 3,
+    content,
+    date: Date.now(),
+    author: "",
+    version: link.version,
+    labels: [],
+    selectionStart: link.selectionStart,
+    selectionEnd: link.selectionEnd,
+  };
+}

tools/links-check/report.ts

@@ -70,6 +70,7 @@ export function printReport(report: Report): Summary {
     console.info(`✅ No broken links found amongst ${total} total.`);
     if (outdated > 0) {
       console.info(`⚠️  Yet there are ${outdated} outdated links. See above.`);
+      console.info(`⚠️  You can use '--fix' flag to automatically update these links with suggestions.`);
     }
   } else {
     console.info(`⁉️  Detected some potentially broken links ${broken.length}/${total}:`);
@@ -83,6 +84,12 @@ export function printReport(report: Report): Summary {
       }
       console.info();
     }
+
+    if (outdated !== broken.length) {
+      console.info(
+        `⚠️  You can use '--fix' flag to automatically update outdated links. Broken links are not going to be changed.`,
+      );
+    }
   }
 
   return {

tools/matrix-bot/convert-to-notes.ts

@@ -1,13 +1,12 @@
 import fs from "node:fs";
 import path from "node:path";
-import type { ISynctexBlockId, Metadata } from "@fluffylabs/links-metadata";
+import type { INoteV3, INotesEnvelopeV3, Metadata } from "@fluffylabs/links-metadata";
 import { fetchMetadata, findLink, parseLink } from "@fluffylabs/links-metadata";
 
 /**
  * Convert messages.json file generated by the tool into Reader notes format
  * so it can be imported or loaded as remote content.
  */
-
 type Message = {
   /** ISO date. */
   date: string;
@@ -19,31 +18,6 @@ type Message = {
   msg: string;
 };
 
-// TODO [ToDr] Note these types could be shared between the `matrix-bot`
-// and the reader, however since we want to avoid having too many shared
-// packages (to simplify publishing to npm) and this type does not really
-// fit the `links-metadata`, I've decided to duplicate it here.
-// Note it's a historical format, so it SHOULD NOT change anyway.
-type NoteV3 = {
-  noteVersion: 3;
-  content: string;
-  /** local-tz timestamp (from Date.now()) */
-  date: number;
-  /** empty for local notes. */
-  author: string;
-  /** Full version number. */
-  version: string;
-  /** Labels. */
-  labels: string[];
-
-  selectionStart: ISynctexBlockId;
-  selectionEnd: ISynctexBlockId;
-};
-type INotesEnvelopeV3 = {
-  version: 3;
-  notes: NoteV3[];
-};
-
 async function main(file = "./output/messages.json") {
   const content = fs.readFileSync(path.resolve(file), "utf-8");
   // note that the file is not a valid JSON as-is (it's appended to),
@@ -54,7 +28,7 @@ async function main(file = "./output/messages.json") {
   // get metadata and synctex data.
   const meta = await fetchMetadata();
 
-  const notes = new Map<string, NoteV3>();
+  const notes = new Map<string, INoteV3>();
   for (const msg of data) {
     const linkData = await findAndParseLink(msg.msg, meta);
     if (linkData === null) {