fix: KVS.getPublicUrl() reads the public URL directly from storage client (#3286)

Removes incorrect implementation `KVS.getPublicUrl()` implementation from `@crawlee/core` and proxies the call to the storage client.

Closes #3272 
Closes #3076

Author: barjin

docs/upgrading/upgrading_v4.md

@@ -107,3 +107,6 @@ The `CrawlingContext.response` property is now of type [`Response`](https://deve
 
 The crawling context in the `FileDownload` crawler no longer includes the `body` and `stream` properties. These can be accessed directly via the `response` property instead, e.g. `context.response.bytes()` or `context.response.body`.
 
+## `KeyValueStore.getPublicUrl` is now async
+
+The `KeyValueStore.getPublicUrl` method is now asynchronous and reads the public URL directly from the storage client.

packages/core/src/crawlers/error_snapshotter.ts

@@ -88,9 +88,9 @@ export class ErrorSnapshotter {
 
             return {
                 screenshotFileName,
-                screenshotFileUrl: screenshotFileName && keyValueStore.getPublicUrl(screenshotFileName),
+                screenshotFileUrl: screenshotFileName && (await keyValueStore.getPublicUrl(screenshotFileName)),
                 htmlFileName,
-                htmlFileUrl: htmlFileName && keyValueStore.getPublicUrl(htmlFileName),
+                htmlFileUrl: htmlFileName && (await keyValueStore.getPublicUrl(htmlFileName)),
             };
         } catch {
             return {};

packages/core/src/storages/key_value_store.ts

@@ -483,10 +483,13 @@ export class KeyValueStore {
 
     /**
      * Returns a file URL for the given key.
+     *
+     * If the record does not exist or has no associated file path (i.e., it is not stored as a file), returns `undefined`.
+     *
+     * @param key The key of the record to generate the public URL for.
      */
-    getPublicUrl(key: string): string {
-        const name = this.name ?? this.config.get('defaultKeyValueStoreId');
-        return `file://${process.cwd()}/storage/key_value_stores/${name}/${key}`;
+    async getPublicUrl(key: string): Promise<string | undefined> {
+        return this.client.getRecordPublicUrl(key);
     }
 
     /**

packages/memory-storage/src/fs/key-value-store/fs.ts

@@ -34,8 +34,9 @@ export class KeyValueFileSystemEntry implements StorageImplementation<InternalKe
             file = await readFile(this.filePath);
         } catch {
             try {
+                const noExtFilePath = resolve(this.storeDirectory, this.rawRecord.key);
                 // Try without extension
-                file = await readFile(resolve(this.storeDirectory, this.rawRecord.key));
+                file = await readFile(noExtFilePath);
                 memoryStorageLog.warning(
                     [
                         `Key-value entry "${this.rawRecord.key}" for store ${basename(
@@ -45,6 +46,7 @@ export class KeyValueFileSystemEntry implements StorageImplementation<InternalKe
                     ].join('\n'),
                 );
                 file = file.toString('utf-8');
+                this.filePath = noExtFilePath;
             } catch {
                 // This is impossible to happen, but just in case
                 throw new Error(`Could not find file at ${this.filePath}`);
@@ -56,6 +58,7 @@ export class KeyValueFileSystemEntry implements StorageImplementation<InternalKe
         return {
             ...this.rawRecord,
             value: file,
+            filePath: this.filePath,
         };
     }
 

packages/memory-storage/src/resource-clients/key-value-store.ts

@@ -32,6 +32,7 @@ export interface InternalKeyRecord {
     value: Buffer | string;
     contentType?: string;
     extension: string;
+    filePath?: string;
 }
 
 export class KeyValueStoreClient extends BaseClient {
@@ -184,6 +185,27 @@ export class KeyValueStoreClient extends BaseClient {
         };
     }
 
+    /**
+     * Generates a public file:// URL for accessing a specific record in the key-value store.
+     *
+     * Returns `undefined` if the record does not exist or has no associated file path (i.e., it is not stored as a file).
+     * @param key The key of the record to generate the public URL for.
+     */
+    async getRecordPublicUrl(key: string): Promise<string | undefined> {
+        s.string().parse(key);
+
+        // Check by id
+        const existingStoreById = await findOrCacheKeyValueStoreByPossibleId(this.client, this.name ?? this.id);
+
+        if (!existingStoreById) {
+            this.throwOnNonExisting(StorageTypes.KeyValueStore);
+        }
+
+        const storageEntry = await existingStoreById.keyValueEntries.get(key)?.get();
+
+        return storageEntry?.filePath;
+    }
+
     /**
      * Tests whether a record with the given key exists in the key-value store without retrieving its value.
      *

packages/types/src/storages.ts

@@ -168,6 +168,7 @@ export interface KeyValueStoreClient {
     delete(): Promise<void>;
     listKeys(options?: KeyValueStoreClientListOptions): Promise<KeyValueStoreClientListData>;
     recordExists(key: string): Promise<boolean>;
+    getRecordPublicUrl(key: string): Promise<string | undefined>;
     getRecord(key: string, options?: KeyValueStoreClientGetRecordOptions): Promise<KeyValueStoreRecord | undefined>;
     setRecord(record: KeyValueStoreRecord, options?: KeyValueStoreRecordOptions): Promise<void>;
     deleteRecord(key: string): Promise<void>;