Prototype: Import Books from URL

[mekarpeles]

Feature Request

This issue is similar to Goodreads "Import by URL"

Problem / Opportunity

On the https://openlibrary.org/account/import page, add a section to import books from URL to a page that contains ISBNS (similar to Bulk Search https://openlibrary.org/search/bulk).

The patron will be asked to enter a URL, the openlibrary front-end uses js to pull down the html of that URL, finds any ISBNs from the page, and then uses the existing import flow to show you your books, allow you to select them, and then finally (a) create a List, (b) add to a list, or (c) add to Want to Read.

Prototype

The first step in this issue would be to write js code that can run on openlibrary.org (as a logged in patron) in the console, that will fetch a URL, extract the ISBNs, show the matching books

This is a good website to test against
https://patrickcollison.com/bookshelf

Proposal

Refer to this map of common Endpoints:
*

Requirements Checklist

Checklist of requirements that need to be satisfied in order for this issue to be closed:

• [ ]

Stakeholders

---

Instructions for Contributors

• Before creating a new branch or pushing up changes to a PR, please first run these commands to ensure your repository is up to date, as the pre-commit bot may add commits to your PRs upstream.

[mekarpeles]

To be clear, the first step of this issue is just a prototype / gist that we can use to extract ISBNs from web pages and display the corresponding Open Library books (e.g. using codepen)

[395ShikharSingh]

Hi @mekarpeles , I’d love to work on this.
I can start working on the JavaScript script (prototype), and if it looks good, we can proceed further.

[395ShikharSingh]

Go to the https://openlibrary.org/ then paste this script in the console.

async function importBooksFromUrl(url) {
    if (!url) {
        console.error("Please provide a URL");
        return;
    }

    console.log(`%c Starting import from: ${url}`, 'font-weight: bold; color: #007bff;');

    const proxyUrl = 'https://api.allorigins.win/get?url=' + encodeURIComponent(url);

    let html = '';
    try {
        const response = await fetch(proxyUrl);
        if (!response.ok) throw new Error(`Status ${response.status}`);
        const data = await response.json();
        html = data.contents;
        console.log(`%c Fetched ${html.length} bytes`, 'color: green;');
    } catch (e) {
        console.error(`%c Fetch failed: ${e.message}`, 'color: red;');
        console.warn("Note: This prototype uses api.allorigins.win. If it fails, the target site might be blocking it or the proxy is down.");
        return;
    }

    const isbnRegex = /\b(?:97[89][-\s]?)?(?:\d[-\s]?){9}[\dXx]\b/g;
    const matches = html.match(isbnRegex) || [];

    const uniqueCanonicalIsbns = new Set();

    matches.forEach(m => {
        const clean = m.replace(/[-\s]/g, '').toUpperCase();
        if (isValidIsbn(clean)) {
            uniqueCanonicalIsbns.add(clean);
        }
    });

    const isbns = Array.from(uniqueCanonicalIsbns);
    console.log(`%c Found ${isbns.length} valid unique ISBNs`, 'color: blue;');

    if (isbns.length === 0) return;

    const chunkSize = 50;
    const bookDetails = [];

    for (let i = 0; i < isbns.length; i += chunkSize) {
        const chunk = isbns.slice(i, i + chunkSize);
        const bibkeys = chunk.map(isbn => `ISBN:${isbn}`).join(',');

        console.log(`Fetching details for batch ${Math.floor(i / chunkSize) + 1}...`);

        try {
            const apiBase = 'https://openlibrary.org/api/books';
            const params = new URLSearchParams({
                bibkeys: bibkeys,
                format: 'json',
                jscmd: 'data'
            });

            const res = await fetch(`${apiBase}?${params.toString()}`);
            const data = await res.json();

            Object.values(data).forEach(book => bookDetails.push(book));

        } catch (e) {
            console.error("API Error:", e);
        }
    }

    console.table(bookDetails.map(b => ({
        Title: b.title,
        Authors: b.authors ? b.authors.map(a => a.name).join(', ') : 'N/A',
        ISBN: b.identifiers?.isbn_13?.[0] || b.identifiers?.isbn_10?.[0] || 'N/A',
        Cover: b.cover?.small ? 'Yes' : 'No',
        Link: b.url
    })));

    console.log(`%c Done Found metadata for ${bookDetails.length} books.`, 'font-weight: bold; color: green;');
    return bookDetails;
}

function isValidIsbn(isbn) {
    if (isbn.length === 10) {
        if (!/^\d{9}[\dX]$/.test(isbn)) return false;
        let sum = 0;
        for (let i = 0; i < 9; i++) sum += parseInt(isbn[i]) * (10 - i);
        const last = isbn[9] === 'X' ? 10 : parseInt(isbn[9]);
        sum += last;
        return sum % 11 === 0;
    } else if (isbn.length === 13) {
        if (!/^\d{13}$/.test(isbn)) return false;
        let sum = 0;
        for (let i = 0; i < 12; i++) {
            sum += parseInt(isbn[i]) * (i % 2 === 0 ? 1 : 3);
        }
        const check = (10 - (sum % 10)) % 10;
        return check === parseInt(isbn[12]);
    }
    return false;
}

call the importBooksFromUrl() function with url and await. Something like :

await importBooksFromUrl('https://patrickcollison.com/bookshelf')

the result would be like:

[mekarpeles]

@395ShikharSingh thank you, this is a step in the right direction!

One change I would make is to batch Open Library API calls using the search.json API which allows you to search for a set of ISBNs.

The version provided makes an individual expensive API request to https://openlibrary.org/api/books per book as opposed to in batches. Open Library has a search.json API defined here: https://openlibrary.org/dev/docs/api/search

Queries can be configured to search for e.g. 50 editions at a time by ISBN: e.g. https://openlibrary.org/search.json?q=isbn:(9780387948241%20OR%209781566199094%20OR%209780061120084%20OR%209780321765723)&fields=key,isbn,title,editions,editions.key

Please update the code to use this bulk search json API instead of https://openlibrary.org/api/books e.g.

This entails:

1. fetching the URL content
2. extracting ISBNs
3. split this into a list of lists (of length 50)
4. extracting the edition key of the matching edition e.g. /editions/OL123M

We also likely want to log some sort of progress, e.g. "{num_isbns} ISBNs found"

After this is working, we can add one more function to case above of creating a list and we've have a working utility we can discuss adding to openlibrary.org! Thank you!

Given the contents of this comment, here is what chatgpt came up with:

async function importBooksFromUrl(url) {
    if (!url) {
        console.error("Please provide a URL");
        return;
    }

    console.log(`%c Starting import from: ${url}`, 'font-weight: bold; color: #007bff;');

    const proxyUrl = 'https://api.allorigins.win/get?url=' + encodeURIComponent(url);

    let html = '';
    try {
        const response = await fetch(proxyUrl);
        if (!response.ok) throw new Error(`Status ${response.status}`);
        const data = await response.json();
        html = data.contents;
        console.log(`%c Fetched ${html.length} bytes`, 'color: green;');
    } catch (e) {
        console.error(`%c Fetch failed: ${e.message}`, 'color: red;');
        return;
    }

    // --- Extract ISBNs ---
    const isbnRegex = /\b(?:97[89][-\s]?)?(?:\d[-\s]?){9}[\dXx]\b/g;
    const matches = html.match(isbnRegex) || [];

    const uniqueIsbns = new Set();
    matches.forEach(m => {
        const clean = m.replace(/[-\s]/g, '').toUpperCase();
        if (isValidIsbn(clean)) uniqueIsbns.add(clean);
    });

    const isbns = Array.from(uniqueIsbns);
    console.log(`%c Found ${isbns.length} valid unique ISBNs`, 'color: blue;');

    if (!isbns.length) return;

    // --- Batch search.json queries ---
    const chunkSize = 50;
    const editions = [];

    for (let i = 0; i < isbns.length; i += chunkSize) {
        const chunk = isbns.slice(i, i + chunkSize);

        const q = `isbn:(${chunk.join(' OR ')})`;
        const params = new URLSearchParams({
            q,
            fields: 'key,isbn,title,editions,editions.key',
            limit: chunk.length
        });

        const url = `https://openlibrary.org/search.json?${params.toString()}`;

        console.log(`Fetching batch ${Math.floor(i / chunkSize) + 1} (${chunk.length} ISBNs)`);

        try {
            const res = await fetch(url);
            if (!res.ok) throw new Error(`Status ${res.status}`);
            const data = await res.json();

data.docs.forEach(doc => {
    const editionDocs = doc.editions?.docs;
    if (!Array.isArray(editionDocs) || editionDocs.length === 0) return;

    editionDocs.forEach(ed => {
        if (!ed.key) return;

        editions.push({
            workKey: doc.key,                 // /works/OL…W
            editionKey: ed.key,               // /books/OL…M
            title: doc.title,
            isbns: doc.isbn || []
        });
    });
});


        } catch (e) {
            console.error("Search API error:", e);
        }
    }

    console.table(editions.map(e => ({
        Title: e.title,
        ISBNs: e.isbn?.join(', ') || 'N/A',
        Edition: e.editionKey
    })));

    console.log(
        `%c Done. Found ${editions.length} editions.`,
        'font-weight: bold; color: green;'
    );

    return editions;
}

// --- ISBN validation unchanged ---
function isValidIsbn(isbn) {
    if (isbn.length === 10) {
        if (!/^\d{9}[\dX]$/.test(isbn)) return false;
        let sum = 0;
        for (let i = 0; i < 9; i++) sum += parseInt(isbn[i]) * (10 - i);
        const last = isbn[9] === 'X' ? 10 : parseInt(isbn[9]);
        return (sum + last) % 11 === 0;
    } else if (isbn.length === 13) {
        if (!/^\d{13}$/.test(isbn)) return false;
        let sum = 0;
        for (let i = 0; i < 12; i++) {
            sum += parseInt(isbn[i]) * (i % 2 === 0 ? 1 : 3);
        }
        return ((10 - (sum % 10)) % 10) === parseInt(isbn[12]);
    }
    return false;
}

[mekarpeles]

async function importBooksFromUrl(url) {
    if (!url) {
        console.error("Please provide a URL");
        return;
    }

    console.log(
        `%cStarting import from: ${url}`,
        'font-weight: bold; color: #007bff;'
    );

    // ---------- Fetch page HTML ----------
    const proxyUrl = 'https://api.allorigins.win/get?url=' + encodeURIComponent(url);
    let html = '';

    try {
        const response = await fetch(proxyUrl);
        if (!response.ok) throw new Error(`Status ${response.status}`);
        const data = await response.json();
        html = data.contents;
        console.log(`%cFetched ${html.length} bytes`, 'color: green;');
    } catch (e) {
        console.error(`%cFetch failed: ${e.message}`, 'color: red;');
        return;
    }

    // ---------- Extract ISBNs ----------
    const isbnRegex = /\b(?:97[89][-\s]?)?(?:\d[-\s]?){9}[\dXx]\b/g;
    const matches = html.match(isbnRegex) || [];

    const uniqueIsbns = new Set();
    matches.forEach(m => {
        const clean = m.replace(/[-\s]/g, '').toUpperCase();
        if (isValidIsbn(clean)) uniqueIsbns.add(clean);
    });

    const isbns = Array.from(uniqueIsbns);
    console.log(
        `%cFound ${isbns.length} valid unique ISBNs`,
        'color: #0d6efd; font-weight: bold;'
    );

    if (!isbns.length) return;

    // ---------- Batch search.json queries ----------
    const chunkSize = 50;
    const editions = [];

    const totalIsbns = isbns.length;
    const totalBatches = Math.ceil(totalIsbns / chunkSize);
    let processedIsbns = 0;

    for (let i = 0; i < isbns.length; i += chunkSize) {
        const chunk = isbns.slice(i, i + chunkSize);
        processedIsbns += chunk.length;

        const batchNum = Math.floor(i / chunkSize) + 1;
        const percentDone = ((processedIsbns / totalIsbns) * 100).toFixed(1);

        console.log(
            `%c[Batch ${batchNum}/${totalBatches}] ` +
            `Querying ${processedIsbns} / ${totalIsbns} ISBNs (${percentDone}%)`,
            'color: #6f42c1; font-weight: bold;'
        );

        const q = `isbn:(${chunk.join(' OR ')})`;
        const params = new URLSearchParams({
            q,
            fields: 'key,isbn,title,editions',
            limit: chunk.length
        });

        const searchUrl = `https://openlibrary.org/search.json?${params.toString()}`;

        try {
            const res = await fetch(searchUrl);
            if (!res.ok) throw new Error(`Status ${res.status}`);
            const data = await res.json();

            let editionsFoundThisBatch = 0;

            data.docs.forEach(doc => {
                const editionDocs = doc.editions?.docs;
                if (!Array.isArray(editionDocs) || editionDocs.length === 0) return;

                editionDocs.forEach(ed => {
                    if (!ed.key) return;

                    editions.push({
                        workKey: doc.key,       // /works/OL…W
                        editionKey: ed.key,    // /books/OL…M
                        title: doc.title,
                        isbns: doc.isbn || []
                    });

                    editionsFoundThisBatch++;
                });
            });

            console.log(
                `%c→ Batch ${batchNum} returned ${data.docs.length} works, ` +
                `${editionsFoundThisBatch} editions`,
                'color: #198754;'
            );

        } catch (e) {
            console.error(
                `%c✖ Batch ${batchNum} failed: ${e.message}`,
                'color: red; font-weight: bold;'
            );
        }
    }

    // ---------- Summary ----------
    console.table(
        editions.map(e => ({
            Title: e.title,
            ISBNs: e.isbns.join(', '),
            Edition: e.editionKey
        }))
    );

    console.log(
        `%cDone. Found ${editions.length} editions.`,
        'font-weight: bold; color: green;'
    );

    return editions;
}

// ---------- ISBN validation ----------
function isValidIsbn(isbn) {
    if (isbn.length === 10) {
        if (!/^\d{9}[\dX]$/.test(isbn)) return false;
        let sum = 0;
        for (let i = 0; i < 9; i++) {
            sum += parseInt(isbn[i], 10) * (10 - i);
        }
        const last = isbn[9] === 'X' ? 10 : parseInt(isbn[9], 10);
        return (sum + last) % 11 === 0;
    }

    if (isbn.length === 13) {
        if (!/^\d{13}$/.test(isbn)) return false;
        let sum = 0;
        for (let i = 0; i < 12; i++) {
            sum += parseInt(isbn[i], 10) * (i % 2 === 0 ? 1 : 3);
        }
        return ((10 - (sum % 10)) % 10) === parseInt(isbn[12], 10);
    }

    return false;
}