fix file content diff check

Author: peterjah

cli/src/lib/website/filesInit.ts

@@ -1,7 +1,6 @@
 import {
   Args,
   Operation,
-  Provider,
   SmartContract,
   strToBytes,
   U32,
@@ -20,7 +19,6 @@ import { FileInit } from './models/FileInit'
 import { Metadata } from './models/Metadata'
 
 import {
-  FILE_TAG,
   fileChunkCountKey,
   fileLocationKey,
   globalMetadataKey,
@@ -291,63 +289,3 @@ class Batch {
     )
   }
 }
-
-/**
- * Filter out pre-stores that are already stored on the blockchain
- * @param provider - the web3 provider
- * @param scAddress - the smart contract address
- * @param fileInits - the pre-stores to filter
- * @returns the pre-stores that are not stored on the blockchain
- */
-export async function filterUselessFileInits(
-  provider: Provider,
-  scAddress: string,
-  fileInits: FileInit[]
-): Promise<FileInit[]> {
-  const fileInitsWithKey = fileInits.map((preStore) => {
-    return {
-      preStore: preStore,
-      totalChunkKey: fileChunkCountKey(preStore.hashLocation),
-    }
-  })
-
-  const batches: {
-    preStore: FileInit
-    totalChunkKey: Uint8Array
-  }[][] = []
-
-  for (let i = 0; i < fileInitsWithKey.length; i += 100) {
-    batches.push(fileInitsWithKey.slice(i, i + 100))
-  }
-
-  const fileInitsToKeep: FileInit[] = []
-
-  const keys = await provider.getStorageKeys(scAddress, FILE_TAG)
-  for (const batch of batches) {
-    // Remove missing keys from the batch and add them to the list of files to keep
-    for (let i = batch.length - 1; i >= 0; i--) {
-      if (!keys.includes(batch[i].totalChunkKey)) {
-        fileInitsToKeep.push(batch[i].preStore)
-        batch.splice(i, 1)
-      }
-    }
-
-    const results = await provider.readStorage(
-      scAddress,
-      batch.map((key) => key.totalChunkKey)
-    )
-
-    for (let i = 0; i < batch.length; i++) {
-      const chunkData = results[i]
-      if (
-        !chunkData ||
-        chunkData.length !== U32.SIZE_BYTE ||
-        U32.fromBytes(chunkData) !== batch[i].preStore.totalChunk
-      ) {
-        fileInitsToKeep.push(batch[i].preStore)
-      }
-    }
-  }
-
-  return fileInitsToKeep
-}

cli/src/lib/website/read.ts

@@ -88,29 +88,17 @@ export async function getFileFromAddress(
     filePath
   )
 
-  const datastoreKeys = []
+  const chunksKeys = []
   for (let i = 0n; i < fileTotalChunks; i++) {
-    datastoreKeys.push(fileChunkKey(filePathHash, i))
+    chunksKeys.push(fileChunkKey(filePathHash, i))
   }
 
-  const rawChunks = (await provider.readStorage(scAddress, datastoreKeys)).map(
-    // allow to return Uint8Array[] instead of (Uint8Array | null)[]
-    (chunk, i) => {
-      if (!chunk || chunk.length === 0) {
-        throw new Error(`file ${filePath} Chunk ${i} not found`)
-      }
-      return chunk
-    }
-  )
-
-  const totalLength = rawChunks.reduce((acc, chunk) => acc + chunk.length, 0)
-  const concatenatedArray = new Uint8Array(totalLength)
+  const fileChunks = await provider.readStorage(scAddress, chunksKeys)
 
-  let offset = 0
-  for (const chunk of rawChunks) {
-    concatenatedArray.set(chunk, offset)
-    offset += chunk.length
-  }
-
-  return concatenatedArray
+  return fileChunks
+    .filter((chunk) => !!chunk)
+    .reduce(
+      (acc, chunk) => Uint8Array.from([...acc, ...chunk]),
+      new Uint8Array()
+    )
 }

cli/src/lib/website/storageKeys.ts

@@ -41,7 +41,7 @@ export function fileChunkKey(
 ): Uint8Array {
   return new Uint8Array([
     ...fileKey(hashLocation),
-    ...CHUNK_NB_TAG,
+    ...CHUNK_TAG,
     ...U32.toBytes(index),
   ])
 }

cli/src/tasks/prepareChunk.ts

@@ -18,7 +18,6 @@ import { FileInit } from '../lib/website/models/FileInit'
 
 import { FileDelete } from '../lib/website/models/FileDelete'
 import { UploadCtx } from './tasks'
-import { filterUselessFileInits } from '../lib/website/filesInit'
 
 /**
  * Create a task to prepare batches from the website file
@@ -34,6 +33,7 @@ export function prepareBatchesTask(): ListrTask {
         ctx.chunkSize,
         ctx.sc
       )
+      ctx.fileInits = fileInits
 
       // check that the provided website directory has correct format
       if (!ctx.skipIndexHtmlCheck && !localFiles.includes('index.html')) {
@@ -59,9 +59,6 @@ export function prepareBatchesTask(): ListrTask {
       }
 
       ctx.batches = batcher(chunks, ctx.chunkSize)
-      ctx.fileInits = ctx.sc
-        ? await filterUselessFileInits(ctx.provider, ctx.sc.address, fileInits)
-        : fileInits
 
       if (ctx.batches.length < 16) {
         for (const batch of ctx.batches) {
@@ -72,14 +69,16 @@ export function prepareBatchesTask(): ListrTask {
         }
       }
 
-      task.output = `Total of ${fileInits.length} files, ${ctx.fileInits.length} require update`
+      task.output = `Total of ${localFiles.length} files, ${fileInits.length} require update`
       task.output = `${ctx.filesToDelete.length} files will be deleted from the smart contract`
       if (ctx.filesToDelete.length < 16) {
         for (const file of ctx.filesToDelete) {
           task.output = `- ${file.location}`
         }
       }
-      task.output = `Total of ${chunks.length} chunks divided into ${ctx.batches.length} batches`
+      if (ctx.batches.length) {
+        task.output = `Total of ${chunks.length} chunks divided into ${ctx.batches.length} batches`
+      }
     },
     rendererOptions: {
       outputBar: Infinity,

cli/src/tasks/prepareUpload.ts

@@ -26,7 +26,7 @@ export function prepareUploadTask(): ListrTask {
         ctx.metadatas.length === 0 &&
         ctx.metadatasToDelete.length === 0
       ) {
-        task.skip('All files are ready for upload, and no metadata changes')
+        task.skip('No changes in files and metadata to upload')
         return
       }
 

cli/tests/fileInit.spec.ts

@@ -1,4 +1,3 @@
-import { describe, it, expect } from '@jest/globals'
 import { createBatches, batchSize } from '../src/lib/website/filesInit'
 import { FileInit } from '../src/lib/website/models/FileInit'
 import { FileDelete } from '../src/lib/website/models/FileDelete'