fix(rune): output serialization bug

Author: dasein108

src/services/ipfs/utils/content.ts

@@ -85,121 +85,121 @@ export const parseArrayLikeToDetails = async (
   cid: string,
   onProgress?: onProgressCallback
 ): Promise<IPFSContentDetails> => {
-  try {
-    if (!content || !content?.result) {
-      return {
-        gateway: true,
-        text: cid.toString(),
-        cid,
-      };
-    }
+  // try {
+  if (!content || !content?.result) {
+    return {
+      gateway: true,
+      text: cid.toString(),
+      cid,
+    };
+  }
 
-    const { result, meta } = content;
+  const { result, meta } = content;
 
-    const mime = meta?.mime;
+  const mime = meta?.mime;
 
-    if (!mime) {
-      return {
-        cid,
-        gateway: true,
-        text: `Can't detect MIME for ${cid.toString()}`,
-      };
-    }
-    const contentType = mimeToBaseContentType(mime);
-    const contentCid = content.cid;
-
-    const response: IPFSContentDetails = {
-      link: `/ipfs/${cid}`,
-      gateway: false,
-      cid: contentCid,
-      type: contentType,
+  if (!mime) {
+    return {
+      cid,
+      gateway: true,
+      text: `Can't detect MIME for ${cid.toString()}`,
     };
+  }
+  const contentType = mimeToBaseContentType(mime);
+  const contentCid = content.cid;
+
+  const response: IPFSContentDetails = {
+    link: `/ipfs/${cid}`,
+    gateway: false,
+    cid: contentCid,
+    type: contentType,
+  };
+
+  if (detectGatewayContentType(mime)) {
+    return { ...response, gateway: true };
+  }
 
-    if (detectGatewayContentType(mime)) {
-      return { ...response, gateway: true };
-    }
+  const rawData =
+    typeof result !== 'string'
+      ? await getResponseResult(result, onProgress)
+      : result;
 
-    const rawData =
-      typeof result !== 'string'
-        ? await getResponseResult(result, onProgress)
-        : result;
+  const isStringData = typeof rawData === 'string';
 
-    const isStringData = typeof rawData === 'string';
+  // console.log(rawData);
+  if (!rawData) {
+    return {
+      ...response,
+      gateway: true,
+      text: `Can't parse content for ${cid.toString()}`,
+    };
+  }
 
-    // console.log(rawData);
-    if (!rawData) {
+  // clarify text-content subtypes
+  if (response.type === 'text') {
+    // render svg as image
+    if (!isStringData && isSvg(Buffer.from(rawData))) {
       return {
         ...response,
-        gateway: true,
-        text: `Can't parse content for ${cid.toString()}`,
+        type: 'image',
+        content: createImgData(rawData, 'image/svg+xml'),
       };
     }
 
-    // clarify text-content subtypes
-    if (response.type === 'text') {
-      // render svg as image
-      if (!isStringData && isSvg(Buffer.from(rawData))) {
-        return {
-          ...response,
-          type: 'image',
-          content: createImgData(rawData, 'image/svg+xml'),
-        };
-      }
-
-      const str = isStringData ? rawData : uint8ArrayToAsciiString(rawData);
-
-      if (str.match(PATTERN_IPFS_HASH)) {
-        return {
-          ...response,
-          type: 'cid',
-          content: str,
-        };
-      }
-      if (str.match(PATTERN_HTTP)) {
-        return {
-          ...response,
-          type: 'link',
-          content: str,
-        };
-      }
-      if (isHtml(str)) {
-        return {
-          ...response,
-          type: 'html',
-          gateway: true,
-          content: cid.toString(),
-        };
-      }
-
-      // TODO: search can bel longer for 42???!
-      // also cover ipns links
+    const str = isStringData ? rawData : uint8ArrayToAsciiString(rawData);
+
+    if (str.match(PATTERN_IPFS_HASH)) {
       return {
         ...response,
-        link: str.length > 42 ? `/ipfs/${cid}` : `/search/${str}`,
-        type: 'text',
-        text: shortenString(str),
+        type: 'cid',
         content: str,
       };
     }
-
-    if (!isStringData) {
-      if (response.type === 'image') {
-        return { ...response, content: createImgData(rawData, mime) }; // file
-      }
-      if (response.type === 'pdf') {
-        return {
-          ...response,
-          content: createObjectURL(rawData, mime),
-          gateway: true,
-        }; // file
-      }
+    if (str.match(PATTERN_HTTP)) {
+      return {
+        ...response,
+        type: 'link',
+        content: str,
+      };
+    }
+    if (isHtml(str)) {
+      return {
+        ...response,
+        type: 'html',
+        gateway: true,
+        content: cid.toString(),
+      };
     }
 
-    return response;
-  } catch (e) {
-    console.log('----parseRawIpfsData', e, cid);
-    return undefined;
+    // TODO: search can bel longer for 42???!
+    // also cover ipns links
+    return {
+      ...response,
+      link: str.length > 42 ? `/ipfs/${cid}` : `/search/${str}`,
+      type: 'text',
+      text: shortenString(str),
+      content: str,
+    };
   }
+
+  if (!isStringData) {
+    if (response.type === 'image') {
+      return { ...response, content: createImgData(rawData, mime) }; // file
+    }
+    if (response.type === 'pdf') {
+      return {
+        ...response,
+        content: createObjectURL(rawData, mime),
+        gateway: true,
+      }; // file
+    }
+  }
+
+  return response;
+  // } catch (e) {
+  //   console.log('----parseRawIpfsData', e, cid);
+  //   return undefined;
+  // }
 };
 
 export const contentToUint8Array = async (

src/services/ipfs/utils/stream.ts

@@ -132,27 +132,32 @@ export const getResponseResult = async (
       return readArray;
     }
 
-    const reader = response[Symbol.asyncIterator]();
-
-    // if (cid === 'QmRqms6Utkk6L4mtyLQXY2spcQ8Pk7fBBTNjvxa9jTNrXp') {
-    //   debugger;
-    // }
-    // eslint-disable-next-line no-restricted-syntax
-    for await (const chunk of reader) {
-      if (chunk instanceof Uint8Array) {
-        chunks.push(chunk);
-        bytesDownloaded += chunk.byteLength;
-        onProgress && onProgress(bytesDownloaded);
+    if (Symbol.asyncIterator in response) {
+      const reader = response[Symbol.asyncIterator]();
+
+      // if (cid === 'QmRqms6Utkk6L4mtyLQXY2spcQ8Pk7fBBTNjvxa9jTNrXp') {
+      //   debugger;
+      // }
+      // eslint-disable-next-line no-restricted-syntax
+      for await (const chunk of reader) {
+        if (chunk instanceof Uint8Array) {
+          chunks.push(chunk);
+          bytesDownloaded += chunk.byteLength;
+          onProgress && onProgress(bytesDownloaded);
+        }
       }
+      const result = uint8ArrayConcat(chunks);
+      return result;
     }
-    const result = uint8ArrayConcat(chunks);
-    return result;
+    return undefined;
   } catch (error) {
     console.error(
       `Error reading stream/iterable.\r\n Probably Hot reload error!`,
       error
     );
 
+    // throw error;
+
     return undefined;
   }
 };

src/services/ipfs/utils/utils-ipfs.ts

@@ -175,16 +175,6 @@ const fetchIPFSContentFromGateway = async (
     method: 'GET',
     signal: controller?.signal,
   });
-  if (cid === 'QmakRbRoKh5Nss8vbg9qnNN2Bcsr7jUX1nbDeMT5xe8xa1') {
-    console.log(
-      '----fetchIPFSContentFromGateway',
-      response,
-      contentUrl,
-      cid,
-      node,
-      controller?.signal
-    );
-  }
   if (response && response.body) {
     // fetch doesn't provide any headers in our case :(
 

src/services/scripting/engine.ts

@@ -2,23 +2,18 @@ import initAsync, { compile } from 'cyb-rune-wasm';
 
 import { v4 as uuidv4 } from 'uuid';
 
-import { CyberClient } from '@cybercongress/cyber-js';
-import {
-  OfflineSigner,
-  SigningCyberClient,
-} from '@cybercongress/cyber-js/build/signingcyberclient';
 import { TabularKeyValues } from 'src/types/data';
 import { keyValuesToObject } from 'src/utils/localStorage';
 
 import { mapObjIndexed } from 'ramda';
+import { removeBrokenUnicode } from 'src/utils/string';
+
 import { extractRuneScript } from './helpers';
-import { RemoteIpfsApi } from '../backend/workers/background/worker';
 
 import {
   ScriptCallback,
   ScriptParticleParams,
   ScriptContext,
-  ScriptMyParticleResult,
   ScriptParticleResult,
   // ScriptMyParticleParams,
   ScriptEntrypoints,
@@ -29,7 +24,6 @@ import {
 } from './types';
 
 import runtimeScript from './rune/runtime.rn';
-import { deserializeString, serializeString } from 'src/utils/string';
 
 const compileConfig = {
   budget: 1_000_000,
@@ -46,23 +40,9 @@ type CompilerParams = {
   config: typeof compileConfig;
 };
 
-// | { name: 'ipfs'; item: AppIPFS }
-// | { name: 'queryClient'; item: CyberClient }
-// | {
-//     name: 'signer';
-//     item: { signer?: OfflineSigner; signingClient: SigningCyberClient };
-//   };
-
 const toRecord = (item: TabularKeyValues) =>
   keyValuesToObject(Object.values(item));
 
-// type EngineDeps = {
-//   ipfs?: RemoteIpfsApi;
-//   queryClient?: CyberClient;
-//   signer?: OfflineSigner;
-//   signingClient?: SigningCyberClient;
-// };
-
 export type LoadParams = {
   entrypoints: ScriptEntrypoints;
   secrets: TabularKeyValues;
@@ -184,7 +164,7 @@ function enigine(): RuneEngine {
         ...outputData,
         error,
         result: result
-          ? JSON.parse(result)
+          ? JSON.parse(removeBrokenUnicode(result))
           : { action: 'error', message: 'No result' },
       };
     } catch (e) {
@@ -233,21 +213,21 @@ function enigine(): RuneEngine {
     const { cid, contentType, content } = params;
     const output = await run(script, {
       funcName: 'personal_processor',
-      funcParams: [cid, contentType, serializeString(content || '')], //params as EntrypointParams,
+      funcParams: [cid, contentType, content], //params as EntrypointParams,
     });
 
     const { action, content: outputContent } = output.result;
 
     if (action === 'error') {
       console.error(
-        `[rune].personalProcessor error: ${params.cid}`,
+        `RUNE: personalProcessor error: ${params.cid}`,
         params,
         output
       );
     }
 
     if (outputContent) {
-      return { ...output.result, content: deserializeString(outputContent) };
+      return { ...output.result, content: outputContent };
     }
 
     return output.result;

src/utils/string.ts

@@ -25,3 +25,9 @@ export function deserializeString(serialized: string): string {
     .replace(/\\\\/g, '\\') // Unescape backslashes
     .replace(/\\!!/g, '#'); // Unescape # cozo comment
 }
+
+const specialCharsRegexe = /\\u\{[a-fA-F0-9]+\}/g;
+
+export function removeBrokenUnicode(string: string): string {
+  return string.replace(specialCharsRegexe, '');
+}