db improvements and optimizations

Author: dd3v

README.md

@@ -16,22 +16,77 @@
 </p>
 
 
-FavBox is a browser extension that enhances and simplifies bookmark management, without depending on external cloud storage or third-party services.
+FavBox is a local-first **experimental** browser extension that enhances and simplifies bookmark management without cloud storage or third-party services. It extends your browser's native bookmarking features.
 
 Key features:
 
 🔄 sync with your browser profile;\
 🔒 does not send your data to any third-party services;\
+🎨 minimalist and clean UI; \
 🏷 supports tags for easy organization;\
-🔍 offers advanced search, sorting, and filtering by tags, domains, and folders;\
-🌁 provides multiple display modes; \
-🌗 light and dark theme.\
-📖 offers page previews for quick reference;\
-🟢 lets you know if a bookmark already exists, to avoid creating duplicates;\
-🗑️ provides a feature to detect broken URLs;\
-⌨️ provides quick access to search with hotkeys "Ctrl + Shift + K"; \
+🔍 provides advanced search, sorting, and filtering capabilities based on tags, domains, folders, and webpage keywords;\
+🌁 multiple display modes; \
+🌗 light and dark theme;\
+🗑️ includes a health check function that detects broken URLs; \
+⌨️ provides quick access to search with hotkeys;  \
+🗒️ includes functionality for creating and managing **local** notes; \
 ❤️ free and open source;
 
+### Concept
+
+![image](concept.png) 
+
+### Implementation
+
+FavBox scans all bookmarks in the browser, then makes requests to the saved pages and extracts data from them such as title, description, image, and meta tags to improve the search. All the data is stored in local storage IndexedDB. The extension also tracks all browser events related to bookmarks and synchronizes the data. It only extends the standard functionality and does not attempt to replace it. You can work with bookmarks both through the extension and the native browser’s built-in bookmark features.
+
+
+FavBox is a fully local application. To keep tags synced across devices, it uses a trick. Since bookmarks are synchronized between devices, to keep tags synchronized, the app adds them to the page title.
+
+For example, if you have a bookmark titled `Google Chrome — Wikipedia`, to save tags across devices, extension appends them to the title like this:
+`Google Chrome — Wikipedia 🏷 #wiki #browser`
+
+This way, your tags become available on other devices without using any cloud services — only through the standard Google Chrome profile sync.
+
+
+```
+├── public                 # Static assets (icons, etc.)
+│   └── icons
+├── src                    # Source code
+│   ├── assets             # Global assets
+│   ├── components         # Common reusable app components
+│   │   └── app
+│   ├── ext                # Browser extension includes main app, popup, content script, and service worker
+│   │   ├── browser        # FavBox app
+│   │   │   ├── components # FavBox components
+│   │   │   ├── layouts   
+│   │   │   └── views
+│   │   ├── content        # Content scripts
+│   │   ├── popup          # Extension PopUp window
+│   │   └── sw             # Service Worker of the browser extension
+│   ├── helpers            # Shared utilities
+│   ├── parser             # Library to parse HTML content
+│   ├── storage            # IndexedDB storage
+│   │   └── idb
+│   └── workers            # JS Workers
+└── tests
+    ├── integration
+    └── unit
+```
+
 ### Building
 1. `pnpm run build` to build into `dist`
-2. Enable dev mode in `chrome://extensions/` and `Load unpacked` extension
+2. Enable dev mode in `chrome://extensions/` and `Load unpacked` extension
+
+### Commands
+
+- **`dev`**  Start development server  
+- **`dev:firefox`**  Firefox development build (WIP)
+- **`build`**  Production build  
+- **`test:unit`** Run unit tests  
+- **`test:integration`**   Run integration tests  
+
+### TODO
+- Use SQLite Wasm for storage (ideal for future experiments)
+- Improve transaction implementation (ensure reliability & better performance)
+- The extension already uses a polyfill to maintain compatibility with other browsers. It would be good to test this in Firefox. (WIP)

app_demo copy.png

@@ -2,7 +2,7 @@
   "manifest_version": 3,
   "name": "FavBox",
   "description": "Clear and modern bookmarking tool.",
-  "version": "2.0",
+  "version": "2.0.1",
   "permissions": [
     "bookmarks",
     "activeTab",

app_demo.png

@@ -2,7 +2,7 @@
     "manifest_version": 3,
     "name": "FavBox",
     "description": "Clear and modern bookmarking tool.",
-    "version": "2.0",
+    "version": "2.0.1",
     "permissions": [
         "bookmarks",
         "activeTab",

concept.png

@@ -2,7 +2,6 @@ import { createApp } from 'vue';
 import '@/assets/app.css';
 import masonry from 'vue-next-masonry';
 import Notifications from 'notiwind';
-import { MotionPlugin } from '@vueuse/motion';
 import FloatingVue from 'floating-vue';
 import router from './router';
 import AppLayout from './layouts/AppLayout.vue';
@@ -11,7 +10,6 @@ import '@zanmato/vue3-treeselect/dist/vue3-treeselect.min.css';
 import '@fontsource-variable/inter';
 
 const app = createApp(AppLayout).use(router).use(masonry).use(Notifications)
-  .use(MotionPlugin)
   .use(FloatingVue);
 
 app.mount('#app');

manifest.chrome.json

@@ -161,7 +161,7 @@ const commands = [
 const paginate = async (skip) => {
   try {
     items.value.push(
-      ...(await attributeStorage.filterAttributesByKeyAndValue(command.value.value, searchTerm.value, skip, 100)),
+      ...(await attributeStorage.filterByKeyAndValue(command.value.value, searchTerm.value, skip, 100)),
     );
   } catch (e) {
     console.error(e);
@@ -245,7 +245,7 @@ const arraySearch = () => {
 
 const dbSearch = async () => {
   try {
-    items.value = await attributeStorage.filterAttributesByKeyAndValue(command.value.value, searchTerm.value, 0, 50);
+    items.value = await attributeStorage.filterByKeyAndValue(command.value.value, searchTerm.value, 0, 50);
   } catch (e) {
     console.error(e);
   }

manifest.firefox.json

@@ -67,29 +67,42 @@ browser.contextMenus.onClicked.addListener((info) => {
 
 // https:// developer.browser.com/docs/extensions/reference/bookmarks/#event-onCreated
 browser.bookmarks.onCreated.addListener(async (id, bookmark) => {
+  console.time(`bookmark-created-${id}`);
   console.warn('🎉 Handle bookmark create..', id, bookmark);
   if (bookmark.url === undefined) {
     console.warn('bad bookmark data', bookmark);
     return;
   }
   let response = null;
-  const [activeTab] = await browser.tabs.query({ active: true });
-  try {
-    console.warn('activeTab', activeTab);
-    console.warn('requesting html from tab', activeTab);
-    const content = await browser.tabs.sendMessage(activeTab.id, { action: 'getHTML' });
-    response = { html: content?.html, error: 0 };
-    console.warn('response from tab', response);
-  } catch (e) {
-    console.log('No tabs. It is weird. Fetching data from internet.. 🌎');
+  let activeTab = null;
+  const { nativeImport } = await browser.storage.session.get('nativeImport');
+  console.log('native import', nativeImport);
+
+  if (nativeImport === true) {
+    console.log('Native browser import. Fetching data from internet.. 🌎');
     response = await fetchHelper.fetch(bookmark.url, 15000);
+  } else {
+    // fetch HTML from active tab (content script)
+    activeTab = await browser.tabs.query({ active: true });
+    try {
+      console.warn('activeTab', activeTab);
+      console.warn('requesting html from tab', activeTab);
+      const content = await browser.tabs.sendMessage(activeTab.id, { action: 'getHTML' });
+      response = { html: content?.html, error: 0 };
+      console.warn('response from tab', response);
+    } catch (e) {
+      console.log('No tabs. It is weird. Fetching data from internet.. 🌎');
+      response = await fetchHelper.fetch(bookmark.url, 15000);
+    }
   }
+
   try {
     if (response === null) {
       throw new Error('No page data: response is null');
     }
-    const entity = await (new MetadataParser(bookmark, response)).getFavboxBookmark();
-    if (entity.image === null && activeTab) {
+    const foldersMap = await bookmarkHelper.buildFoldersMap();
+    const entity = await (new MetadataParser(bookmark, response, foldersMap)).getFavboxBookmark();
+    if (!nativeImport && entity.image === null && activeTab) {
       try {
         console.warn('📸 No image, take a screenshot', activeTab);
         const screenshot = await browser.tabs.captureVisibleTab(activeTab.windowId, { format: 'jpeg', quality: 10 });
@@ -98,48 +111,55 @@ browser.bookmarks.onCreated.addListener(async (id, bookmark) => {
         console.error('📸', e);
       }
     }
-    console.warn('Entity', entity);
-    const r = await bookmarkStorage.create(entity);
-    console.log('🎉 Bookmark has been created..', r);
+    console.log('🔖 Entity', entity);
+    await bookmarkStorage.create(entity);
+    await attributeStorage.create(entity);
     refreshUserInterface();
+    console.log('🎉 Bookmark has been created..');
   } catch (e) {
     console.error('🎉', e, id, bookmark);
   }
+  console.timeEnd(`bookmark-created-${id}`);
 });
 
-// https://developer.browser.com/docs/extensions/reference/bookmarks/#event-onChanged
+// https://developer.chrome.com/docs/extensions/reference/bookmarks/#event-onChanged
 browser.bookmarks.onChanged.addListener(async (id, changeInfo) => {
+  console.time(`bookmark-changed-${id}`);
   try {
-    console.log('🔄 Bookmark has been updated..', id, changeInfo);
     const [bookmark] = await browser.bookmarks.get(id);
-
-    const folderTree = await bookmarkHelper.getFoldersTreeByBookmark(id);
+    // folder
     if (!bookmark.url) {
       console.warn('changeInfo', changeInfo, bookmark);
-      await bookmarkStorage.updateFolders(bookmark, folderTree);
-    } else {
-      const toUpdate = {
+      console.warn('Folder', bookmark);
+      await bookmarkStorage.updateFolderNameByFolderId(bookmark.id, bookmark.title);
+      console.log('🔄 Folder has been updated..', id, changeInfo);
+    }
+    // bookmark
+    if (bookmark.url) {
+      await bookmarkStorage.update(id, {
         title: tagHelper.getTitle(changeInfo.title),
         tags: tagHelper.getTags(changeInfo.title),
         url: bookmark.url,
         updatedAt: new Date().toISOString(),
-      };
-      await bookmarkStorage.update(id, toUpdate);
-      console.log('🔀', bookmark, toUpdate, toUpdate.tags);
-      await attributeStorage.refreshTags();
+      });
+      console.log('🔄 Bookmark has been updated..', id, changeInfo);
     }
   } catch (e) {
     console.error('🔄', e, id, changeInfo);
   }
   try {
-    // browser.runtime.sendMessage({ action: 'refresh' });
+    await attributeStorage.refreshTags();
+    await attributeStorage.refreshFolders();
+    refreshUserInterface();
   } catch (e) {
-    console.error('Refresh app UI on change', e);
+    console.error('🔄 Error updating attributes', e);
   }
+  console.timeEnd(`bookmark-changed-${id}`);
 });
 
-// https://developer.browser.com/docs/extensions/reference/bookmarks/#event-onMoved
+// https://developer.chrome.com/docs/extensions/reference/bookmarks/#event-onMoved
 browser.bookmarks.onMoved.addListener(async (id, moveInfo) => {
+  console.time(`bookmark-moved-${id}`);
   try {
     const [folder] = await browser.bookmarks.get(moveInfo.parentId);
     console.log('🗂 Bookmark has been moved..', id, moveInfo, folder);
@@ -149,21 +169,26 @@ browser.bookmarks.onMoved.addListener(async (id, moveInfo) => {
       folderId: folder.id,
       updatedAt: new Date().toISOString(),
     });
+    await attributeStorage.refreshFolders();
+    refreshUserInterface();
   } catch (e) {
     console.error('🗂', e, id, moveInfo);
   }
-  refreshUserInterface();
+  console.timeEnd(`bookmark-moved-${id}`);
 });
 
-// https://developer.browser.com/docs/extensions/reference/bookmarks/#event-onRemoved
+// https://developer.chrome.com/docs/extensions/reference/bookmarks/#event-onRemoved
 browser.bookmarks.onRemoved.addListener(async (id, removeInfo) => {
+  console.time(`bookmark-removed-${id}`);
   console.log('🗑️ Handle remove bookmark..', id, removeInfo);
+  // folder has been deleted..
   if (removeInfo.node.children !== undefined) {
     try {
       const items = bookmarkHelper.getAllBookmarksFromNode(removeInfo.node);
       const bookmarksToRemove = items.map((bookmark) => bookmark.id);
       if (bookmarksToRemove.length) {
         const total = await bookmarkStorage.removeByIds(bookmarksToRemove);
+        await attributeStorage.refresh();
         console.log('🗑️ Folder has been removed..', total, id, removeInfo);
       }
       refreshUserInterface();
@@ -172,17 +197,20 @@ browser.bookmarks.onRemoved.addListener(async (id, removeInfo) => {
     }
     return;
   }
+  // single bookmark has been deleted..
   try {
     const bookmark = await bookmarkStorage.getById(id);
     if (!bookmark) {
       throw new Error(`Bookmark with ID ${id} not found in storage.`);
     }
     await bookmarkStorage.remove(id);
+    await attributeStorage.remove(bookmark);
     refreshUserInterface();
     console.log('🗑️ Bookmark has been removed..', id, removeInfo);
   } catch (e) {
     console.error('🗑️', e);
   }
+  console.timeEnd(`bookmark-removed-${id}`);
 });
 
 // https://bugs.chromium.org/p/chromium/issues/detail?id=1185241
@@ -207,6 +235,18 @@ browser.runtime.onMessage.addListener((message, sender, sendResponse) => {
   return true;
 });
 
+// https://developer.chrome.com/docs/extensions/reference/api/bookmarks#event-onImportBegan
+browser.bookmarks.onImportBegan.addListener(() => {
+  console.log('📄 Import bookmarks started');
+  browser.storage.session.set({ nativeImport: true });
+});
+
+// https://developer.chrome.com/docs/extensions/reference/api/bookmarks#event-onImportEnded
+chrome.bookmarks.onImportEnded.addListener(() => {
+  console.log('📄 Import bookmarks ended');
+  browser.storage.session.set({ nativeImport: false });
+});
+
 function refreshUserInterface() {
   try {
     browser.runtime.sendMessage({ action: 'refresh' });

src/ext/browser/app.js

@@ -7,7 +7,7 @@ import MetadataParser from '@/parser/metadata';
 import bookmarkHelper from '@/helpers/bookmark';
 
 const sync = async () => {
-  console.time('Execution time');
+  console.time('Sync time');
   await initStorage();
   const bookmarkStorage = new BookmarkStorage();
   const attributeStorage = new AttributeStorage();
@@ -23,13 +23,15 @@ const sync = async () => {
     return;
   }
   await browser.storage.session.set({ status: false });
+  const foldersMap = await bookmarkHelper.buildFoldersMap();
+  console.log('Folders map:', foldersMap);
   const bookmarksIterator = await bookmarkHelper.iterateBookmarks();
   let batch = [];
   let processed = 0;
   for await (const b of bookmarksIterator) {
     batch.push(b);
     processed += 1;
-    if (batch.length % 100 === 0 || processed === browserTotal) {
+    if (batch.length % 150 === 0 || processed === browserTotal) {
       try {
         const browserBookmarkKeyList = batch.map((i) => i.id);
         const extBookmarksKeyList = await bookmarkStorage.getIds(browserBookmarkKeyList);
@@ -44,7 +46,7 @@ const sync = async () => {
           return { bookmark, response };
         }));
         const parseResult = await Promise.all(
-          httpResults.map(({ bookmark, response }) => (new MetadataParser(bookmark, response)).getFavboxBookmark()),
+          httpResults.map(({ bookmark, response }) => (new MetadataParser(bookmark, response, foldersMap)).getFavboxBookmark()),
         );
         console.time('DB execution time');
         console.warn(parseResult);
@@ -66,7 +68,7 @@ const sync = async () => {
     }
   }
   await browser.storage.session.set({ status: true });
-  console.timeEnd('Execution time');
+  console.timeEnd('Sync time');
 };
 
 export default sync;