From 7b09085e06e75051af805c131a34b358c2bdb8ef Mon Sep 17 00:00:00 2001 From: Henri Jamet Date: Mon, 8 Sep 2025 16:10:53 +0200 Subject: [PATCH 1/4] Add API functions for content management in Obsidian client - Introduced `deleteContentByType` to delete content of a specified type from the Khoj server. - Added `uploadContentBatch` to handle batch uploads of files to the Khoj content endpoint. - Updated `updateContentIndex` to utilize the new API functions for content deletion and batch uploads, improving error handling and file processing logic. --- src/interface/obsidian/src/api.ts | 30 +++++++ src/interface/obsidian/src/utils.ts | 126 +++++++++++++--------------- 2 files changed, 90 insertions(+), 66 deletions(-) create mode 100644 src/interface/obsidian/src/api.ts diff --git a/src/interface/obsidian/src/api.ts b/src/interface/obsidian/src/api.ts new file mode 100644 index 000000000..6e648fef7 --- /dev/null +++ b/src/interface/obsidian/src/api.ts @@ -0,0 +1,30 @@ +export async function deleteContentByType(khojUrl: string, khojApiKey: string, contentType: string): Promise { + // Deletes all content of a given type on Khoj server for Obsidian client + const response = await fetch(`${khojUrl}/api/content/type/${contentType}?client=obsidian`, { + method: 'DELETE', + headers: khojApiKey ? { 'Authorization': `Bearer ${khojApiKey}` } : {}, + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`Failed to delete content type ${contentType}: ${response.status} ${text}`); + } +} + +export async function uploadContentBatch(khojUrl: string, khojApiKey: string, method: 'PUT' | 'PATCH', files: { blob: Blob, path: string }[]): Promise { + // Uploads a batch of files to Khoj content endpoint + const formData = new FormData(); + files.forEach(fileItem => { formData.append('files', fileItem.blob, fileItem.path); }); + + const response = await fetch(`${khojUrl}/api/content?client=obsidian`, { + method: method, + headers: khojApiKey ? { 'Authorization': `Bearer ${khojApiKey}` } : {}, + body: formData, + }); + + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`Failed to upload batch: ${response.status} ${text}`); + } + + return await response.text(); +} diff --git a/src/interface/obsidian/src/utils.ts b/src/interface/obsidian/src/utils.ts index 4a4faae4d..1a8f1f37a 100644 --- a/src/interface/obsidian/src/utils.ts +++ b/src/interface/obsidian/src/utils.ts @@ -60,6 +60,8 @@ export const supportedImageFilesTypes = fileTypeToExtension.image; export const supportedBinaryFileTypes = fileTypeToExtension.pdf.concat(supportedImageFilesTypes); export const supportedFileTypes = fileTypeToExtension.markdown.concat(supportedBinaryFileTypes); +import { deleteContentByType, uploadContentBatch } from './api'; + export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map, regenerate: boolean = false, userTriggered: boolean = false): Promise> { // Get all markdown, pdf files in the vault console.log(`Khoj: Updating Khoj content index...`) @@ -91,6 +93,7 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las let fileData = []; let currentBatchSize = 0; const MAX_BATCH_SIZE = 10 * 1024 * 1024; // 10MB max batch size + const CHUNK_FILE_COUNT = 50; // Number of files per logical chunk when uploading by type let currentBatch = []; for (const file of files) { @@ -135,85 +138,76 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las // Delete all files of enabled content types first if regenerating let error_message = null; - const contentTypesToDelete = []; + const contentTypesToDelete: string[] = []; if (regenerate) { - // Mark content types to delete based on user sync file type settings if (setting.syncFileType.markdown) contentTypesToDelete.push('markdown'); if (setting.syncFileType.pdf) contentTypesToDelete.push('pdf'); if (setting.syncFileType.images) contentTypesToDelete.push('image'); } - for (const contentType of contentTypesToDelete) { - const response = await fetch(`${setting.khojUrl}/api/content/type/${contentType}?client=obsidian`, { - method: "DELETE", - headers: { - 'Authorization': `Bearer ${setting.khojApiKey}`, - } - }); - if (!response.ok) { - error_message = "❗️Failed to clear existing content index"; - fileData = []; + + // Perform deletions sequentially to avoid rate limiting + try { + for (const contentType of contentTypesToDelete) { + console.log(`Khoj: Starting deletion of ${contentType} files...`); + await deleteContentByType(setting.khojUrl, setting.khojApiKey, contentType); + console.log(`Khoj: ${contentType} files deleted successfully.`); } + } catch (err) { + console.error('Khoj: Error while deleting content types:', err); + error_message = "❗️Failed to clear existing content index"; + fileData = []; } // Iterate through all indexable files in vault, 10Mb batch at a time let responses: string[] = []; - for (const batch of fileData) { - // Create multipart form data with all files in batch - const formData = new FormData(); - batch.forEach(fileItem => { formData.append('files', fileItem.blob, fileItem.path) }); - - // Call Khoj backend to sync index with updated files in vault - const method = regenerate ? "PUT" : "PATCH"; - const response = await fetch(`${setting.khojUrl}/api/content?client=obsidian`, { - method: method, - headers: { - 'Authorization': `Bearer ${setting.khojApiKey}`, - }, - body: formData, - }); + if (!error_message) { + // Group files by type for chunked processing + const filesByType: Record = { + markdown: [], + pdf: [], + image: [], + }; - if (!response.ok) { - if (response.status === 429) { - let response_text = await response.text(); - if (response_text.includes("Too much data")) { - const errorFragment = document.createDocumentFragment(); - errorFragment.appendChild(document.createTextNode("❗️Exceeded data sync limits. To resolve this either:")); - const bulletList = document.createElement('ul'); - - const limitFilesItem = document.createElement('li'); - const settingsPrefixText = document.createTextNode("Limit files to sync from "); - const settingsLink = document.createElement('a'); - settingsLink.textContent = "Khoj settings"; - settingsLink.href = "#"; - settingsLink.addEventListener('click', (e) => { - e.preventDefault(); - openKhojPluginSettings(); - }); - limitFilesItem.appendChild(settingsPrefixText); - limitFilesItem.appendChild(settingsLink); - bulletList.appendChild(limitFilesItem); - - const upgradeItem = document.createElement('li'); - const upgradeLink = document.createElement('a'); - upgradeLink.href = `${setting.khojUrl}/settings#subscription`; - upgradeLink.textContent = 'Upgrade your subscription'; - upgradeLink.target = '_blank'; - upgradeItem.appendChild(upgradeLink); - bulletList.appendChild(upgradeItem); - errorFragment.appendChild(bulletList); - error_message = errorFragment; - } else { - error_message = `❗️Failed to sync your content with Khoj server. Requests were throttled. Upgrade your subscription or try again later.`; + for (const batch of fileData) { + for (const fileItem of batch) { + const ext = fileItem.path.split('.').pop()?.toLowerCase() ?? ''; + if (fileTypeToExtension.markdown.includes(ext)) filesByType.markdown.push(fileItem); + else if (fileTypeToExtension.pdf.includes(ext)) filesByType.pdf.push(fileItem); + else if (fileTypeToExtension.image.includes(ext)) filesByType.image.push(fileItem); + else filesByType.markdown.push(fileItem); + } + } + + const method: 'PUT' | 'PATCH' = regenerate ? 'PUT' : 'PATCH'; + + for (const typeKey of Object.keys(filesByType)) { + const allFilesOfType = filesByType[typeKey] as { blob: Blob, path: string }[]; + if (allFilesOfType.length === 0) continue; + + // Split into logical chunks of CHUNK_FILE_COUNT + const totalBatches = Math.ceil(allFilesOfType.length / CHUNK_FILE_COUNT); + for (let i = 0; i < totalBatches; i++) { + const startIdx = i * CHUNK_FILE_COUNT; + const endIdx = Math.min(startIdx + CHUNK_FILE_COUNT, allFilesOfType.length); + const chunk = allFilesOfType.slice(startIdx, endIdx); + console.log(`Khoj: Indexing ${typeKey} files: batch ${i + 1} of ${totalBatches}...`); + try { + const resultText = await uploadContentBatch(setting.khojUrl, setting.khojApiKey, method, chunk); + responses.push(resultText); + console.log(`Khoj: Successfully indexed ${typeKey} files: batch ${i + 1} of ${totalBatches}.`); + } catch (err: any) { + console.error(`Khoj: Failed to upload ${typeKey} batch ${i + 1}:`, err); + // Surface user-friendly error based on server response text when available + if (err.message && err.message.includes('429')) { + error_message = `❗️Requests were throttled. Upgrade your subscription or try again later.`; + } else { + error_message = `❗️Failed to sync all your content with Khoj server. Error: ${err.message ?? String(err)}`; + } + // Stop processing further batches on error + break; } - break; - } else if (response.status === 404) { - error_message = `❗️Could not connect to Khoj server. Ensure you can connect to it.`; - break; - } else { - error_message = `❗️Failed to sync all your content with Khoj server. Raise issue on Khoj Discord or Github\nError: ${response.statusText}`; } - } else { - responses.push(await response.text()); + if (error_message) break; } } From ae41e01b265f5de24751b97e412e59944c166d76 Mon Sep 17 00:00:00 2001 From: Henri Jamet Date: Mon, 8 Sep 2025 16:33:37 +0200 Subject: [PATCH 2/4] Add estimated cloud storage metrics to settings - Introduced a new setting for displaying estimated cloud storage usage based on files configured for sync. - Implemented a progress indicator and text display for the estimation process. - Added a utility function to calculate used and total storage bytes based on user plan (free or premium). - Enhanced error handling for storage estimation failures. --- src/interface/obsidian/src/settings.ts | 39 +++++++++++++++++++ src/interface/obsidian/src/utils.ts | 52 ++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) diff --git a/src/interface/obsidian/src/settings.ts b/src/interface/obsidian/src/settings.ts index 4a07ca0c7..8cdf74a84 100644 --- a/src/interface/obsidian/src/settings.ts +++ b/src/interface/obsidian/src/settings.ts @@ -341,6 +341,45 @@ export class KhojSettingTab extends PluginSettingTab { indexVaultSetting = indexVaultSetting.setDisabled(false); }) ); + // Estimated Cloud Storage (client-side estimation) + const storageSetting = new Setting(containerEl) + .setName('Estimated Cloud Storage (estimation)') + .setDesc('Estimated storage usage based on files configured for sync. This is a client-side estimation.') + .then(() => { }); + + // Create custom elements: progress and text + const progressEl = document.createElement('progress'); + progressEl.value = 0; + progressEl.max = 1; + progressEl.style.width = '100%'; + const progressText = document.createElement('span'); + progressText.textContent = 'Calculating...'; + storageSetting.descEl.appendChild(progressEl); + storageSetting.descEl.appendChild(progressText); + + // Bind update method + (this as any).updateStorageDisplay = async () => { + // Show calculating state + progressEl.removeAttribute('value'); + progressText.textContent = 'Calculating...'; + try { + const { calculateVaultSyncMetrics } = await import('./utils'); + const metrics = await calculateVaultSyncMetrics(this.app.vault, this.plugin.settings); + const usedMB = (metrics.usedBytes / (1024 * 1024)); + const totalMB = (metrics.totalBytes / (1024 * 1024)); + const usedStr = `${usedMB.toFixed(1)} Mo`; + const totalStr = `${totalMB.toFixed(0)} Mo`; + progressEl.value = metrics.usedBytes; + progressEl.max = metrics.totalBytes; + progressText.textContent = `${usedStr} / ${totalStr}`; + } catch (err) { + console.error('Khoj: Failed to update storage display', err); + progressText.textContent = 'Estimation unavailable'; + } + }; + + // Call initial update + (this as any).updateStorageDisplay(); } private connectStatusIcon() { diff --git a/src/interface/obsidian/src/utils.ts b/src/interface/obsidian/src/utils.ts index 1a8f1f37a..647885c7c 100644 --- a/src/interface/obsidian/src/utils.ts +++ b/src/interface/obsidian/src/utils.ts @@ -600,6 +600,58 @@ export function getLinkToEntry(sourceFiles: TFile[], chosenFile: string, chosenE } } +/** + * Calculate estimated vault sync metrics (used and total bytes). + * This is a client-side estimation based on the configured sync file types and folders. + * The storage limit is determined from the backend-provided `setting.userInfo?.is_active` flag: + * - if true => premium limit (500 MB) + * - otherwise => free limit (10 MB) + * This avoids client-side heuristics and relies on server-provided user info. + */ +export async function calculateVaultSyncMetrics(vault: Vault, setting: KhojSetting): Promise<{ usedBytes: number, totalBytes: number }> { + try { + const files = vault.getFiles() + .filter(file => supportedFileTypes.includes(file.extension)) + .filter(file => { + if (fileTypeToExtension.markdown.includes(file.extension)) return setting.syncFileType.markdown; + if (fileTypeToExtension.pdf.includes(file.extension)) return setting.syncFileType.pdf; + if (fileTypeToExtension.image.includes(file.extension)) return setting.syncFileType.images; + return false; + }) + .filter(file => { + if (setting.syncFolders.length === 0) return true; + return setting.syncFolders.some(folder => + file.path.startsWith(folder + '/') || file.path === folder + ); + }); + + const usedBytes = files.reduce((acc, file) => acc + (file.stat?.size ?? 0), 0); + + // Default to free plan limit + const FREE_LIMIT = 10 * 1024 * 1024; // 10 MB + const PAID_LIMIT = 500 * 1024 * 1024; // 500 MB + let totalBytes = FREE_LIMIT; + + // Determine plan from backend-provided user info. Use FREE_LIMIT as default when info missing. + try { + if (setting.userInfo && setting.userInfo.is_active === true) { + totalBytes = PAID_LIMIT; + } else { + totalBytes = FREE_LIMIT; + } + } catch (err) { + // Defensive: on any unexpected error, fall back to free limit + console.warn('Khoj: Error reading userInfo.is_active, defaulting to free limit', err); + totalBytes = FREE_LIMIT; + } + + return { usedBytes, totalBytes }; + } catch (err) { + console.error('Khoj: Error calculating vault sync metrics:', err); + return { usedBytes: 0, totalBytes: 10 * 1024 * 1024 }; + } +} + export async function fetchChatModels(settings: KhojSetting): Promise { if (!settings.connectedToBackend || !settings.khojUrl) { return []; From 062133f1dbadfe95beb866d1d83a57ba137308d9 Mon Sep 17 00:00:00 2001 From: Henri Jamet Date: Mon, 8 Sep 2025 17:04:02 +0200 Subject: [PATCH 3/4] Enhance sync progress UI and updateContentIndex function - Added a progress indicator and text display for the sync operation in the settings. - Modified the `updateContentIndex` function to accept an optional progress callback, allowing real-time updates on file processing. - Implemented error handling for progress updates to ensure robustness during sync operations. - Created a new progress bar for the Force Sync operation, improving user feedback during lengthy processes. --- src/interface/obsidian/src/settings.ts | 74 ++++++++++++++++++++++---- src/interface/obsidian/src/utils.ts | 31 ++++++++++- 2 files changed, 94 insertions(+), 11 deletions(-) diff --git a/src/interface/obsidian/src/settings.ts b/src/interface/obsidian/src/settings.ts index 8cdf74a84..0ee1ea993 100644 --- a/src/interface/obsidian/src/settings.ts +++ b/src/interface/obsidian/src/settings.ts @@ -308,7 +308,7 @@ export class KhojSettingTab extends PluginSettingTab { button.removeCta(); indexVaultSetting = indexVaultSetting.setDisabled(true); - // Show indicator for indexing in progress + // Show indicator for indexing in progress (animated text) const progress_indicator = window.setInterval(() => { if (button.buttonEl.innerText === 'Updating 🌑') { button.setButtonText('Updating 🌘'); @@ -330,15 +330,55 @@ export class KhojSettingTab extends PluginSettingTab { }, 300); this.plugin.registerInterval(progress_indicator); - this.plugin.settings.lastSync = await updateContentIndex( - this.app.vault, this.plugin.settings, this.plugin.settings.lastSync, true, true - ); + // Obtain sync progress elements by id (created below) + const syncProgressEl = document.getElementById('khoj-sync-progress') as HTMLProgressElement | null; + const syncProgressText = document.getElementById('khoj-sync-progress-text') as HTMLElement | null; - // Reset button once index is updated - window.clearInterval(progress_indicator); - button.setButtonText('Update'); - button.setCta(); - indexVaultSetting = indexVaultSetting.setDisabled(false); + if (syncProgressEl && syncProgressText) { + syncProgressEl.style.display = ''; + syncProgressText.style.display = ''; + syncProgressText.textContent = 'Syncing... 0 / ? files'; + syncProgressEl.value = 0; + syncProgressEl.max = 1; + } + + // Define progress callback + const onProgress = (progress: { processed: number, total: number }) => { + try { + const { processed, total } = progress; + const el = document.getElementById('khoj-sync-progress') as HTMLProgressElement | null; + const txt = document.getElementById('khoj-sync-progress-text') as HTMLElement | null; + if (!el || !txt) return; + el.max = Math.max(total, 1); + el.value = Math.min(processed, el.max); + txt.textContent = `Syncing... ${processed} / ${total} files`; + } catch (err) { + console.warn('Khoj: Error updating sync progress UI', err); + } + }; + + try { + this.plugin.settings.lastSync = await updateContentIndex( + this.app.vault, this.plugin.settings, this.plugin.settings.lastSync, true, true, onProgress + ); + } finally { + // Cleanup: hide sync progress UI and refresh storage estimation + const el = document.getElementById('khoj-sync-progress') as HTMLProgressElement | null; + const txt = document.getElementById('khoj-sync-progress-text') as HTMLElement | null; + if (el) el.style.display = 'none'; + if (txt) txt.style.display = 'none'; + try { + (this as any).updateStorageDisplay(); + } catch (err) { + console.warn('Khoj: Failed to refresh storage display after sync', err); + } + + // Reset animated text and button state + window.clearInterval(progress_indicator); + button.setButtonText('Update'); + button.setCta(); + indexVaultSetting = indexVaultSetting.setDisabled(false); + } }) ); // Estimated Cloud Storage (client-side estimation) @@ -347,7 +387,7 @@ export class KhojSettingTab extends PluginSettingTab { .setDesc('Estimated storage usage based on files configured for sync. This is a client-side estimation.') .then(() => { }); - // Create custom elements: progress and text + // Create custom elements: progress and text for storage estimation const progressEl = document.createElement('progress'); progressEl.value = 0; progressEl.max = 1; @@ -357,6 +397,20 @@ export class KhojSettingTab extends PluginSettingTab { storageSetting.descEl.appendChild(progressEl); storageSetting.descEl.appendChild(progressText); + // Create second progress bar for Force Sync operation (hidden by default) + const syncProgressEl = document.createElement('progress'); + syncProgressEl.id = 'khoj-sync-progress'; + syncProgressEl.value = 0; + syncProgressEl.max = 1; + syncProgressEl.style.width = '100%'; + syncProgressEl.style.display = 'none'; + const syncProgressText = document.createElement('span'); + syncProgressText.id = 'khoj-sync-progress-text'; + syncProgressText.textContent = ''; + syncProgressText.style.display = 'none'; + storageSetting.descEl.appendChild(syncProgressEl); + storageSetting.descEl.appendChild(syncProgressText); + // Bind update method (this as any).updateStorageDisplay = async () => { // Show calculating state diff --git a/src/interface/obsidian/src/utils.ts b/src/interface/obsidian/src/utils.ts index 647885c7c..cdd54deaf 100644 --- a/src/interface/obsidian/src/utils.ts +++ b/src/interface/obsidian/src/utils.ts @@ -62,7 +62,14 @@ export const supportedFileTypes = fileTypeToExtension.markdown.concat(supportedB import { deleteContentByType, uploadContentBatch } from './api'; -export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map, regenerate: boolean = false, userTriggered: boolean = false): Promise> { +export async function updateContentIndex( + vault: Vault, + setting: KhojSetting, + lastSync: Map, + regenerate: boolean = false, + userTriggered: boolean = false, + onProgress?: (progress: { processed: number, total: number }) => void +): Promise> { // Get all markdown, pdf files in the vault console.log(`Khoj: Updating Khoj content index...`) const files = vault.getFiles() @@ -89,6 +96,20 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las let countOfFilesToDelete = 0; lastSync = lastSync.size > 0 ? lastSync : new Map(); + // Compute total number of files that will be processed (index + delete) + // This uses the same logic as the processing loop: skip files not modified + // when not regenerating. + let totalIndexCandidates = 0; + for (const file of files) { + if (!regenerate && file.stat.mtime < (lastSync.get(file) ?? 0)) continue; + totalIndexCandidates++; + } + const totalDeleteCandidates = Array.from(lastSync.keys()).filter(f => !files.includes(f)).length; + const totalFilesToProcess = totalIndexCandidates + totalDeleteCandidates; + + // Progress counter + let processedCount = 0; + // Add all files to index as multipart form data let fileData = []; let currentBatchSize = 0; @@ -195,6 +216,14 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las const resultText = await uploadContentBatch(setting.khojUrl, setting.khojApiKey, method, chunk); responses.push(resultText); console.log(`Khoj: Successfully indexed ${typeKey} files: batch ${i + 1} of ${totalBatches}.`); + // Update progress after successful batch upload + processedCount += chunk.length; + try { + if (onProgress) onProgress({ processed: processedCount, total: totalFilesToProcess }); + } catch (err) { + // Callback errors should not break the sync process + console.warn('Khoj: onProgress callback threw an error', err); + } } catch (err: any) { console.error(`Khoj: Failed to upload ${typeKey} batch ${i + 1}:`, err); // Surface user-friendly error based on server response text when available From f476f57c467850a19d8118adb64f3a031c940aa1 Mon Sep 17 00:00:00 2001 From: Henri Jamet Date: Mon, 8 Sep 2025 17:17:15 +0200 Subject: [PATCH 4/4] Enhance sync progress functionality and UI - Updated the sync progress UI to include a display for the current content type being synced (e.g., Images, PDFs, Markdowns). - Modified the `onProgress` callback in `updateContentIndex` to accept an optional `contentType` parameter for better user feedback during sync operations. - Added error handling for the `updateStorageDisplay` method to ensure robustness during setting changes. - Improved the overall user experience by providing clearer sync progress information. --- src/interface/obsidian/src/settings.ts | 36 +++++++++++++++++++++++--- src/interface/obsidian/src/utils.ts | 4 +-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/interface/obsidian/src/settings.ts b/src/interface/obsidian/src/settings.ts index 0ee1ea993..571cff515 100644 --- a/src/interface/obsidian/src/settings.ts +++ b/src/interface/obsidian/src/settings.ts @@ -227,6 +227,7 @@ export class KhojSettingTab extends PluginSettingTab { .onChange(async (value) => { this.plugin.settings.syncFileType.markdown = value; await this.plugin.saveSettings(); + try { if (typeof (this as any).updateStorageDisplay === 'function') (this as any).updateStorageDisplay(); } catch (e) { console.warn('Khoj: updateStorageDisplay error', e); } })); // Add setting to sync images @@ -238,6 +239,7 @@ export class KhojSettingTab extends PluginSettingTab { .onChange(async (value) => { this.plugin.settings.syncFileType.images = value; await this.plugin.saveSettings(); + try { if (typeof (this as any).updateStorageDisplay === 'function') (this as any).updateStorageDisplay(); } catch (e) { console.warn('Khoj: updateStorageDisplay error', e); } })); // Add setting to sync PDFs @@ -249,6 +251,7 @@ export class KhojSettingTab extends PluginSettingTab { .onChange(async (value) => { this.plugin.settings.syncFileType.pdf = value; await this.plugin.saveSettings(); + try { if (typeof (this as any).updateStorageDisplay === 'function') (this as any).updateStorageDisplay(); } catch (e) { console.warn('Khoj: updateStorageDisplay error', e); } })); // Add setting for sync interval @@ -286,6 +289,7 @@ export class KhojSettingTab extends PluginSettingTab { this.plugin.settings.syncFolders.push(folder); this.plugin.saveSettings(); this.updateFolderList(folderListEl); + try { if (typeof (this as any).updateStorageDisplay === 'function') (this as any).updateStorageDisplay(); } catch (e) { console.warn('Khoj: updateStorageDisplay error', e); } } }); modal.open(); @@ -333,25 +337,41 @@ export class KhojSettingTab extends PluginSettingTab { // Obtain sync progress elements by id (created below) const syncProgressEl = document.getElementById('khoj-sync-progress') as HTMLProgressElement | null; const syncProgressText = document.getElementById('khoj-sync-progress-text') as HTMLElement | null; + const syncProgressType = document.getElementById('khoj-sync-progress-type') as HTMLElement | null; if (syncProgressEl && syncProgressText) { syncProgressEl.style.display = ''; syncProgressText.style.display = ''; + if (syncProgressType) syncProgressType.style.display = 'none'; syncProgressText.textContent = 'Syncing... 0 / ? files'; syncProgressEl.value = 0; syncProgressEl.max = 1; } - // Define progress callback - const onProgress = (progress: { processed: number, total: number }) => { + // Define progress callback (includes optional contentType) + const onProgress = (progress: { processed: number, total: number, contentType?: string }) => { try { - const { processed, total } = progress; + const { processed, total, contentType } = progress; const el = document.getElementById('khoj-sync-progress') as HTMLProgressElement | null; const txt = document.getElementById('khoj-sync-progress-text') as HTMLElement | null; + const typeEl = document.getElementById('khoj-sync-progress-type') as HTMLElement | null; if (!el || !txt) return; el.max = Math.max(total, 1); el.value = Math.min(processed, el.max); txt.textContent = `Syncing... ${processed} / ${total} files`; + if (typeEl) { + let label = ''; + if (contentType === 'image') label = 'Images'; + else if (contentType === 'pdf') label = 'PDFs'; + else if (contentType === 'markdown') label = 'Markdowns'; + else if (contentType) label = contentType; + if (label) { + typeEl.textContent = `(${label})`; + typeEl.style.display = ''; + } else { + typeEl.style.display = 'none'; + } + } } catch (err) { console.warn('Khoj: Error updating sync progress UI', err); } @@ -365,8 +385,10 @@ export class KhojSettingTab extends PluginSettingTab { // Cleanup: hide sync progress UI and refresh storage estimation const el = document.getElementById('khoj-sync-progress') as HTMLProgressElement | null; const txt = document.getElementById('khoj-sync-progress-text') as HTMLElement | null; + const typeEl = document.getElementById('khoj-sync-progress-type') as HTMLElement | null; if (el) el.style.display = 'none'; if (txt) txt.style.display = 'none'; + if (typeEl) typeEl.style.display = 'none'; try { (this as any).updateStorageDisplay(); } catch (err) { @@ -408,8 +430,15 @@ export class KhojSettingTab extends PluginSettingTab { syncProgressText.id = 'khoj-sync-progress-text'; syncProgressText.textContent = ''; syncProgressText.style.display = 'none'; + // Element to display the current content type being synced (e.g. Images, PDFs, Markdowns) + const syncProgressType = document.createElement('span'); + syncProgressType.id = 'khoj-sync-progress-type'; + syncProgressType.textContent = ''; + syncProgressType.style.display = 'none'; + syncProgressType.style.marginLeft = '8px'; storageSetting.descEl.appendChild(syncProgressEl); storageSetting.descEl.appendChild(syncProgressText); + storageSetting.descEl.appendChild(syncProgressType); // Bind update method (this as any).updateStorageDisplay = async () => { @@ -558,6 +587,7 @@ export class KhojSettingTab extends PluginSettingTab { this.plugin.settings.syncFolders = this.plugin.settings.syncFolders.filter(f => f !== folder); await this.plugin.saveSettings(); this.updateFolderList(containerEl); + try { if (typeof (this as any).updateStorageDisplay === 'function') (this as any).updateStorageDisplay(); } catch (e) { console.warn('Khoj: updateStorageDisplay error', e); } }); }); } diff --git a/src/interface/obsidian/src/utils.ts b/src/interface/obsidian/src/utils.ts index cdd54deaf..5d58ddb3d 100644 --- a/src/interface/obsidian/src/utils.ts +++ b/src/interface/obsidian/src/utils.ts @@ -68,7 +68,7 @@ export async function updateContentIndex( lastSync: Map, regenerate: boolean = false, userTriggered: boolean = false, - onProgress?: (progress: { processed: number, total: number }) => void + onProgress?: (progress: { processed: number, total: number, contentType?: string }) => void ): Promise> { // Get all markdown, pdf files in the vault console.log(`Khoj: Updating Khoj content index...`) @@ -219,7 +219,7 @@ export async function updateContentIndex( // Update progress after successful batch upload processedCount += chunk.length; try { - if (onProgress) onProgress({ processed: processedCount, total: totalFilesToProcess }); + if (onProgress) onProgress({ processed: processedCount, total: totalFilesToProcess, contentType: typeKey }); } catch (err) { // Callback errors should not break the sync process console.warn('Khoj: onProgress callback threw an error', err);