diff --git a/src/index.ts b/src/index.ts index 853c15d..2ebb850 100644 --- a/src/index.ts +++ b/src/index.ts @@ -57,7 +57,9 @@ import { v4 as uuid4 } from "uuid"; import { HashNotFoundError, ValidationError } from "./error"; import { LruCache } from "./lru"; import { + Entries, RawRemarkable, + SchemaVersion, type BackgroundFilter, type CollectionContent, type Content, @@ -65,7 +67,6 @@ import { type Metadata, type Orientation, type RawEntry, - type RawListEntry, type RawRemarkableApi, type RequestMethod, type SimpleEntry, @@ -86,15 +87,15 @@ export type { CPageUUID, DocumentContent, DocumentMetadata, + Entries, FileType, KeyboardMetadata, Metadata, Orientation, PageTag, RawEntry, - RawFileEntry, - RawListEntry, RawRemarkableApi, + SchemaVersion, SimpleEntry, Tag, TemplateContent, @@ -405,7 +406,7 @@ export interface RemarkableApi { * @remarks * If this fails validation and you still want to get the content, you can use * the low-level api to get the raw text of the `.content` file in the - * `RawListEntry` for this hash. + * `RawEntry` for this hash. * * @param hash - the hash of the item to get content for * @returns the content @@ -421,7 +422,7 @@ export interface RemarkableApi { * @remarks * If this fails validation and you still want to get the content, you can use * the low-level api to get the raw text of the `.metadata` file in the - * `RawListEntry` for this hash. + * `RawEntry` for this hash. * * @param hash - the hash of the item to get metadata for * @returns the metadata @@ -725,7 +726,7 @@ class Remarkable implements RemarkableApi { /** the same cache that underlies the raw api, allowing us to modify it */ readonly #cache: Map; readonly raw: RawRemarkable; - #lastHashGen: readonly [string, number] | undefined; + #lastHashGen: readonly [string, number, SchemaVersion] | undefined; constructor( userToken: string, @@ -746,7 +747,7 @@ class Remarkable implements RemarkableApi { async #getRootHash( refresh: boolean = false, - ): Promise { + ): Promise { if (refresh || this.#lastHashGen === undefined) { this.#lastHashGen = await this.raw.getRootHash(); } @@ -755,7 +756,9 @@ class Remarkable implements RemarkableApi { async #putRootHash(hash: string, generation: number): Promise { try { - this.#lastHashGen = await this.raw.putRootHash(hash, generation); + const [rootHash, gen] = await this.raw.putRootHash(hash, generation); + const [, , schemaVersion] = this.#lastHashGen!; // guaranteed to be set + this.#lastHashGen = [rootHash, gen, schemaVersion]; } catch (ex) { // if we hit a generation error, invalidate our cached generation if (ex instanceof GenerationError) { @@ -803,7 +806,7 @@ class Remarkable implements RemarkableApi { } async #convertEntry({ hash, id }: SimpleEntry): Promise { - const entries = await this.raw.getEntries(hash); + const { entries } = await this.raw.getEntries(hash); const metaEnt = entries.find((ent) => ent.id.endsWith(".metadata")); const contentEnt = entries.find((ent) => ent.id.endsWith(".content")); if (metaEnt === undefined) { @@ -875,12 +878,12 @@ class Remarkable implements RemarkableApi { async listIds(refresh: boolean = false): Promise { const [hash] = await this.#getRootHash(refresh); - const entries = await this.raw.getEntries(hash); + const { entries } = await this.raw.getEntries(hash); return entries.map(({ id, hash }) => ({ id, hash })); } async getContent(hash: string): Promise { - const entries = await this.raw.getEntries(hash); + const { entries } = await this.raw.getEntries(hash); const [cont] = entries.filter((e) => e.id.endsWith(".content")); if (cont === undefined) { throw new Error(`couldn't find contents for hash ${hash}`); @@ -890,7 +893,7 @@ class Remarkable implements RemarkableApi { } async getMetadata(hash: string): Promise { - const entries = await this.raw.getEntries(hash); + const { entries } = await this.raw.getEntries(hash); const [meta] = entries.filter((e) => e.id.endsWith(".metadata")); if (meta === undefined) { throw new Error(`couldn't find metadata for hash ${hash}`); @@ -900,7 +903,7 @@ class Remarkable implements RemarkableApi { } async getPdf(hash: string): Promise { - const entries = await this.raw.getEntries(hash); + const { entries } = await this.raw.getEntries(hash); const [pdf] = entries.filter((e) => e.id.endsWith(".pdf")); if (pdf === undefined) { throw new Error(`couldn't find pdf for hash ${hash}`); @@ -910,7 +913,7 @@ class Remarkable implements RemarkableApi { } async getEpub(hash: string): Promise { - const entries = await this.raw.getEntries(hash); + const { entries } = await this.raw.getEntries(hash); const [epub] = entries.filter((e) => e.id.endsWith(".epub")); if (epub === undefined) { throw new Error(`couldn't find epub for hash ${hash}`); @@ -920,7 +923,7 @@ class Remarkable implements RemarkableApi { } async getDocument(hash: string): Promise { - const entries = await this.raw.getEntries(hash); + const { entries } = await this.raw.getEntries(hash); const zip = new JSZip(); for (const entry of entries) { // TODO if this is .metadata we might want to assert type === "DocumentType" @@ -1004,7 +1007,7 @@ class Remarkable implements RemarkableApi { [metadataEntry, uploadMetadata], [pagedataEntry, uploadPagedata], [fileEntry, uploadFile], - [rootHash, generation], + [rootHash, generation, schemaVersion], ] = await Promise.all([ this.raw.putContent(`${id}.content`, content), this.raw.putMetadata(`${id}.metadata`, metadata), @@ -1015,14 +1018,13 @@ class Remarkable implements RemarkableApi { ]); // now fetch root entries and upload this file entry - const [[collectionEntry, uploadCollection], rootEntries] = + const [[collectionEntry, uploadCollection], { entries: rootEntries }] = await Promise.all([ - this.raw.putEntries(id, [ - contentEntry, - metadataEntry, - pagedataEntry, - fileEntry, - ]), + this.raw.putEntries( + id, + [contentEntry, metadataEntry, pagedataEntry, fileEntry], + schemaVersion, + ), this.raw.getEntries(rootHash), ]); @@ -1031,6 +1033,7 @@ class Remarkable implements RemarkableApi { const [rootEntry, uploadRoot] = await this.raw.putEntries( "root", rootEntries, + schemaVersion, ); // before updating the root hash, first upload everything @@ -1098,7 +1101,7 @@ class Remarkable implements RemarkableApi { const [ [contentEntry, uploadContent], [metadataEntry, uploadMetadata], - [rootHash, generation], + [rootHash, generation, schemaVersion], ] = await Promise.all([ this.raw.putContent(`${id}.content`, content), this.raw.putMetadata(`${id}.metadata`, metadata), @@ -1106,9 +1109,9 @@ class Remarkable implements RemarkableApi { ]); // now fetch root entries and upload this file entry - const [[collectionEntry, uploadCollection], rootEntries] = + const [[collectionEntry, uploadCollection], { entries: rootEntries }] = await Promise.all([ - this.raw.putEntries(id, [contentEntry, metadataEntry]), + this.raw.putEntries(id, [contentEntry, metadataEntry], schemaVersion), this.raw.getEntries(rootHash), ]); @@ -1117,6 +1120,7 @@ class Remarkable implements RemarkableApi { const [rootEntry, uploadRoot] = await this.raw.putEntries( "root", rootEntries, + schemaVersion, ); // before updating the root hash, first upload everything @@ -1162,8 +1166,9 @@ class Remarkable implements RemarkableApi { id: string, hash: string, update: Partial, - ): Promise<[RawListEntry, Promise<[void, void]>]> { - const entries = await this.raw.getEntries(hash); + schemaVersion: SchemaVersion, + ): Promise<[RawEntry, Promise<[void, void]>]> { + const { entries } = await this.raw.getEntries(hash); const contInd = entries.findIndex((ent) => ent.id.endsWith(".content")); const contEntry = entries[contInd]; if (contEntry === undefined) { @@ -1176,7 +1181,11 @@ class Remarkable implements RemarkableApi { cont, ); entries[contInd] = newContEntry; - const [result, uploadEntries] = await this.raw.putEntries(id, entries); + const [result, uploadEntries] = await this.raw.putEntries( + id, + entries, + schemaVersion, + ); const upload = Promise.all([uploadCont, uploadEntries]); return [result, upload]; } @@ -1188,8 +1197,9 @@ class Remarkable implements RemarkableApi { expectedType: "DocumentType" | "CollectionType" | "TemplateType", refresh: boolean, ): Promise { - const [rootHash, generation] = await this.#getRootHash(refresh); - const entries = await this.raw.getEntries(rootHash); + const [rootHash, generation, schemaVersion] = + await this.#getRootHash(refresh); + const { entries } = await this.raw.getEntries(rootHash); const hashInd = entries.findIndex((ent) => ent.hash === hash); const hashEnt = entries[hashInd]; if (hashEnt === undefined) { @@ -1197,7 +1207,7 @@ class Remarkable implements RemarkableApi { } const [[newEnt, uploadEnt], meta] = await Promise.all([ - this.#editContentRaw(hashEnt.id, hash, update), + this.#editContentRaw(hashEnt.id, hash, update, schemaVersion), this.getMetadata(hash), ]); if (meta.type !== expectedType) { @@ -1207,7 +1217,11 @@ class Remarkable implements RemarkableApi { } entries[hashInd] = newEnt; - const [rootEntry, uploadRoot] = await this.raw.putEntries("root", entries); + const [rootEntry, uploadRoot] = await this.raw.putEntries( + "root", + entries, + schemaVersion, + ); await Promise.all([uploadEnt, uploadRoot]); await this.#putRootHash(rootEntry.hash, generation); @@ -1245,8 +1259,9 @@ class Remarkable implements RemarkableApi { id: string, hash: string, update: Partial, - ): Promise<[RawListEntry, Promise<[void, void]>]> { - const entries = await this.raw.getEntries(hash); + schemaVersion: SchemaVersion, + ): Promise<[RawEntry, Promise<[void, void]>]> { + const { entries } = await this.raw.getEntries(hash); const metaInd = entries.findIndex((ent) => ent.id.endsWith(".metadata")); const metaEntry = entries[metaInd]; if (metaEntry === undefined) { @@ -1259,7 +1274,11 @@ class Remarkable implements RemarkableApi { meta, ); entries[metaInd] = newMetaEntry; - const [result, uploadEntries] = await this.raw.putEntries(id, entries); + const [result, uploadEntries] = await this.raw.putEntries( + id, + entries, + schemaVersion, + ); const upload = Promise.all([uploadMeta, uploadEntries]); return [result, upload]; } @@ -1269,8 +1288,9 @@ class Remarkable implements RemarkableApi { update: Partial, refresh: boolean = false, ): Promise { - const [rootHash, generation] = await this.#getRootHash(refresh); - const entries = await this.raw.getEntries(rootHash); + const [rootHash, generation, schemaVersion] = + await this.#getRootHash(refresh); + const { entries } = await this.raw.getEntries(rootHash); const hashInd = entries.findIndex((ent) => ent.hash === hash); const hashEnt = entries[hashInd]; if (hashEnt === undefined) { @@ -1280,9 +1300,14 @@ class Remarkable implements RemarkableApi { hashEnt.id, hash, update, + schemaVersion, ); entries[hashInd] = newEnt; - const [rootEntry, uploadRoot] = await this.raw.putEntries("root", entries); + const [rootEntry, uploadRoot] = await this.raw.putEntries( + "root", + entries, + schemaVersion, + ); await Promise.all([uploadEnt, uploadRoot]); @@ -1343,8 +1368,9 @@ class Remarkable implements RemarkableApi { ); } - const [rootHash, generation] = await this.#getRootHash(refresh); - const entries = await this.raw.getEntries(rootHash); + const [rootHash, generation, schemaVersion] = + await this.#getRootHash(refresh); + const { entries } = await this.raw.getEntries(rootHash); const hashSet = new Set(hashes); const toUpdate: RawEntry[] = []; @@ -1355,7 +1381,9 @@ class Remarkable implements RemarkableApi { } const resolved = await Promise.all( - toUpdate.map(({ id, hash }) => this.#editMetaRaw(id, hash, { parent })), + toUpdate.map(({ id, hash }) => + this.#editMetaRaw(id, hash, { parent }, schemaVersion), + ), ); const uploads: Promise<[void, void]>[] = []; const result: Record = {}; @@ -1368,6 +1396,7 @@ class Remarkable implements RemarkableApi { const [rootEntry, uploadRoot] = await this.raw.putEntries( "root", newEntries, + schemaVersion, ); await Promise.all([Promise.all(uploads), uploadRoot]); @@ -1397,8 +1426,9 @@ class Remarkable implements RemarkableApi { // should only go one step) to track all hashes encountered // NOTE that we could increase the cache in this process, or it's possible // for other calls to increase the cache with misc values. - let entries = [await this.raw.getEntries(rootHash)]; - let nextEntries: Promise[] = []; + const base = await this.raw.getEntries(rootHash); + let entries = [base.entries]; + let nextEntries: Promise[] = []; while (entries.length) { for (const entryList of entries) { for (const { hash, type } of entryList) { @@ -1408,7 +1438,8 @@ class Remarkable implements RemarkableApi { } } } - entries = await Promise.all(nextEntries); + const resolved = await Promise.all(nextEntries); + entries = resolved.map((ent) => ent.entries); nextEntries = []; } for (const key of toDelete) { diff --git a/src/raw.ts b/src/raw.ts index 6ebf372..aebeced 100644 --- a/src/raw.ts +++ b/src/raw.ts @@ -34,6 +34,9 @@ export type UploadMimeType = | "application/epub+zip" | "folder"; +/** the schema version */ +export type SchemaVersion = 3 | 4; + /** an simple entry without any extra information */ export interface SimpleEntry { /** the document id */ @@ -50,9 +53,9 @@ export interface SimpleEntry { * files, the high level entry will have the same hash and id as the low-level * entry for that collection. */ -export interface RawListEntry { - /** collection type (80000000) */ - type: 80000000; +export interface RawEntry { + /** 80000000 for schema 3 collection type or 0 for schema 4 or schema 3 files or */ + type: 80000000 | 0; /** the hash of the collection this points to */ hash: string; /** the unique id of the collection */ @@ -63,25 +66,23 @@ export interface RawListEntry { size: number; } -/** the low-level entry for a single file */ -export interface RawFileEntry { - /** file type (0) */ - type: 0; - /** the hash of the file this points to */ - hash: string; - /** the unique id of the file */ - id: string; - /** the number of subfiles, always zero */ - subfiles: 0; - /** the size of the file in bytes */ - size: number; -} - -/** a low-level stored entry */ -export type RawEntry = RawListEntry | RawFileEntry; /** the type of files reMarkable supports */ export type FileType = "epub" | "pdf" | "notebook"; +/** + * a parsed entries file + * + * id and size are defined for schema 4 but not for 3 + */ +export interface Entries { + /** the raw entries in the file */ + entries: RawEntry[]; + /** the id of this entry, only specified for schema 4 */ + id?: string; + /** the recursive size of this entry, only specified for schema 4 */ + size?: number; +} + /** a tag for an entry */ export interface Tag { /** the name of the tag */ @@ -759,7 +760,7 @@ export interface RawRemarkableApi { * * @returns the root hash and the current generation */ - getRootHash(): Promise<[string, number]>; + getRootHash(): Promise<[string, number, SchemaVersion]>; /** * get the raw binary data associated with a hash @@ -789,7 +790,7 @@ export interface RawRemarkableApi { * @param hash - the hash to get entries for * @returns the entries */ - getEntries(hash: string): Promise; + getEntries(hash: string): Promise; /** * get the parsed and validated `Content` of a content hash @@ -852,25 +853,19 @@ export interface RawRemarkableApi { * @param bytes - the bytes to upload * @returns the new entry and a promise to finish the upload */ - putFile( - id: string, - bytes: Uint8Array, - ): Promise<[RawFileEntry, Promise]>; + putFile(id: string, bytes: Uint8Array): Promise<[RawEntry, Promise]>; /** the same as {@link putFile | `putFile`} but with caching for text */ - putText(id: string, content: string): Promise<[RawFileEntry, Promise]>; + putText(id: string, content: string): Promise<[RawEntry, Promise]>; /** the same as {@link putText | `putText`} but with extra validation for Content */ - putContent( - id: string, - content: Content, - ): Promise<[RawFileEntry, Promise]>; + putContent(id: string, content: Content): Promise<[RawEntry, Promise]>; /** the same as {@link putText | `putText`} but with extra validation for Metadata */ putMetadata( id: string, metadata: Metadata, - ): Promise<[RawFileEntry, Promise]>; + ): Promise<[RawEntry, Promise]>; /** * put a set of entries to make an entry list file @@ -890,7 +885,8 @@ export interface RawRemarkableApi { putEntries( id: string, entries: RawEntry[], - ): Promise<[RawListEntry, Promise]>; + schemaVersion: SchemaVersion, + ): Promise<[RawEntry, Promise]>; /** * upload a file to the reMarkable cloud using the simple api @@ -930,6 +926,29 @@ interface AuthedFetch { ): Promise; } +function parseRawEntryLine(line: string): RawEntry { + const [hash, type, id, subfiles, size] = line.split(":"); + if ( + hash === undefined || + type === undefined || + id === undefined || + subfiles === undefined || + size === undefined + ) { + throw new Error(`line '${line}' was not formatted correctly`); + } else if (type === "80000000" || type === "0") { + return { + hash, + type: type === "0" ? 0 : 80000000, + id, + subfiles: parseInt(subfiles), + size: parseInt(size), + }; + } else { + throw new Error(`line '${line}' was not formatted correctly`); + } +} + export class RawRemarkable implements RawRemarkableApi { readonly #authedFetch: AuthedFetch; readonly #rawHost: string; @@ -958,20 +977,20 @@ export class RawRemarkable implements RawRemarkableApi { } /** make an authorized request to remarkable */ - async getRootHash(): Promise<[string, number]> { + async getRootHash(): Promise<[string, number, SchemaVersion]> { const res = await this.#authedFetch("GET", `${this.#rawHost}/sync/v4/root`); const raw = await res.text(); const loaded = JSON.parse(raw) as unknown; if (!rootHash.guardAssert(loaded)) throw Error("invalid root hash"); const { hash, generation, schemaVersion } = loaded; - if (schemaVersion !== 3) { + if (schemaVersion !== 3 && schemaVersion !== 4) { throw new Error(`schema version ${schemaVersion} not supported`); } else if (!Number.isSafeInteger(generation)) { throw new Error( `generation ${generation} was not a safe integer; please file a bug report`, ); } else { - return [hash, generation]; + return [hash, generation, schemaVersion]; } } @@ -1018,42 +1037,35 @@ export class RawRemarkable implements RawRemarkableApi { } } - async getEntries(hash: string): Promise { + async getEntries(hash: string): Promise { const rawFile = await this.getText(hash); const [version, ...rest] = rawFile.slice(0, -1).split("\n"); - if (version != "3") { - throw new Error(`schema version ${version} not supported`); + if (version === "3") { + return { entries: rest.map(parseRawEntryLine) }; + } else if (version === "4") { + const [info, ...remaining] = rest; + if (!info) throw new Error("missing info line for schema version 4"); + const [lead, id, count, size] = info.split(":"); + if ( + lead !== "0" || + id === undefined || + count === undefined || + size === undefined + ) { + throw new Error( + `schema 4 info line '${info}' was not formatted correctly`, + ); + } + const entries = remaining.map(parseRawEntryLine); + if (parseInt(count) !== entries.length) { + throw new Error( + `schema 4 expected ${count} entries, but found ${entries.length}`, + ); + } else { + return { entries, id, size: parseInt(size) }; + } } else { - return rest.map((line) => { - const [hash, type, id, subfiles, size] = line.split(":"); - if ( - hash === undefined || - type === undefined || - id === undefined || - subfiles === undefined || - size === undefined - ) { - throw new Error(`line '${line}' was not formatted correctly`); - } else if (type === "80000000") { - return { - hash, - type: 80000000, - id, - subfiles: parseInt(subfiles), - size: parseInt(size), - }; - } else if (type === "0" && subfiles === "0") { - return { - hash, - type: 0, - id, - subfiles: 0, - size: parseInt(size), - }; - } else { - throw new Error(`line '${line}' was not formatted correctly`); - } - }); + throw new Error(`schema version ${version} not supported`); } } @@ -1150,9 +1162,9 @@ export class RawRemarkable implements RawRemarkableApi { async putFile( id: string, bytes: Uint8Array, - ): Promise<[RawFileEntry, Promise]> { + ): Promise<[RawEntry, Promise]> { const hash = await digest(bytes); - const res: RawFileEntry = { + const res: RawEntry = { id, hash, type: 0, @@ -1162,10 +1174,7 @@ export class RawRemarkable implements RawRemarkableApi { return [res, this.#putFile(hash, id, bytes)]; } - async putText( - id: string, - text: string, - ): Promise<[RawFileEntry, Promise]> { + async putText(id: string, text: string): Promise<[RawEntry, Promise]> { const enc = new TextEncoder(); const bytes = enc.encode(text); const [ent, upload] = await this.putFile(id, bytes); @@ -1181,7 +1190,7 @@ export class RawRemarkable implements RawRemarkableApi { async putContent( id: string, content: Content, - ): Promise<[RawFileEntry, Promise]> { + ): Promise<[RawEntry, Promise]> { if (!id.endsWith(".content")) { throw new Error(`id ${id} did not end with '.content'`); } else { @@ -1192,7 +1201,7 @@ export class RawRemarkable implements RawRemarkableApi { async putMetadata( id: string, metadata: Metadata, - ): Promise<[RawFileEntry, Promise]> { + ): Promise<[RawEntry, Promise]> { if (!id.endsWith(".metadata")) { throw new Error(`id ${id} did not end with '.metadata'`); } else { @@ -1203,7 +1212,8 @@ export class RawRemarkable implements RawRemarkableApi { async putEntries( id: string, entries: RawEntry[], - ): Promise<[RawListEntry, Promise]> { + schemaVersion: SchemaVersion, + ): Promise<[RawEntry, Promise]> { // NOTE collections have a special hash function, the hash of their // contents, so this needs to be different entries.sort((a, b) => a.id.localeCompare(b.id)); @@ -1216,14 +1226,21 @@ export class RawRemarkable implements RawRemarkableApi { const hash = await digest(hashBuff); const size = entries.reduce((acc, ent) => acc + ent.size, 0); - const records = ["3\n"]; + const records = [`${schemaVersion}\n`]; + if (schemaVersion === 3) { + // noop + } else if (schemaVersion === 4) { + records.push(`0:${id}:${entries.length}:${size}\n`); + } else { + throw new Error(`unsupported schema version ${schemaVersion as number}`); + } for (const { hash, type, id, subfiles, size } of entries) { records.push(`${hash}:${type}:${id}:${subfiles}:${size}\n`); } - const res: RawListEntry = { + const res: RawEntry = { id, hash, - type: 80000000, + type: schemaVersion > 3 ? 0 : 80000000, subfiles: entries.length, size, };