rewrite download procedure, test passed, clean up

tinoryj · tinoryj · commit 3f0f020b025d · 2020-03-17T21:33:52.000+08:00
diff --git a/TEDStore/src/CMakeLists.txt b/TEDStore/src/CMakeLists.txt
@@ -9,11 +9,8 @@ endif()
 set(OPENSSL_INCLUDE_PATH ../lib/openssl/include)
 set(OPENSSL_LIBRARY_PATH ../lib/openssl/)
 
-# set(LEVELDB_INCLUDE_PATH ../lib/leveldb/include)
-# set(LEVELDB_LIBRARY_PATH ../lib/leveldb/build)
-
-set(LEVELDB_INCLUDE_PATH ~/Library/leveldb-1.22/include)
-set(LEVELDB_LIBRARY_PATH ~/Library/leveldb-1.22/build)
+set(LEVELDB_INCLUDE_PATH ../lib/leveldb/include)
+set(LEVELDB_LIBRARY_PATH ../lib/leveldb/build)
 
 include_directories(${BOOST_INCLUDE_DIR} ${LEVELDB_INCLUDE_PATH} ${OPENSSL_INCLUDE_PATH} ../include)
 link_directories(${BOOST_LIBRARY_DIR} ${LEVELDB_LIBRARY_PATH} ${OPENSSL_LIBRARY_PATH} util)
diff --git a/TEDStore/src/client/keyClient.cpp b/TEDStore/src/client/keyClient.cpp
@@ -9,25 +9,6 @@ struct timeval timeendKey;
 struct timeval timestartKeySocket;
 struct timeval timeendKeySocket;
 
-void PRINT_BYTE_ARRAY_KEYC(
-    FILE* file, void* mem, uint32_t len)
-{
-    if (!mem || !len) {
-        fprintf(file, "\n( null )\n");
-        return;
-    }
-    uint8_t* array = (uint8_t*)mem;
-    fprintf(file, "%u bytes:\n{\n", len);
-    uint32_t i = 0;
-    for (i = 0; i < len - 1; i++) {
-        fprintf(file, "0x%x, ", array[i]);
-        if (i % 8 == 7)
-            fprintf(file, "\n");
-    }
-    fprintf(file, "0x%x ", array[i]);
-    fprintf(file, "\n}\n");
-}
-
 keyClient::keyClient(Sender* senderObjTemp)
 {
     inputMQ_ = new messageQueue<Data_t>;
@@ -222,18 +203,7 @@ bool keyClient::keyExchange(u_char* batchHashList, int batchNumber, u_char* batc
 
 bool keyClient::encodeChunk(Data_t& newChunk)
 {
-    // cout << "original data" << endl;
-    PRINT_BYTE_ARRAY_KEYC(stdout, newChunk.chunk.logicData, newChunk.chunk.logicDataSize);
     bool statusChunk = cryptoObj_->encryptChunk(newChunk.chunk);
-    // cout << "original key" << endl;
-    // PRINT_BYTE_ARRAY_KEYC(stdout, newChunk.chunk.encryptKey, CHUNK_ENCRYPT_KEY_SIZE);
-    // cout << "encrypted data" << endl;
-    // PRINT_BYTE_ARRAY_KEYC(stdout, newChunk.chunk.logicData, newChunk.chunk.logicDataSize);
-    // cryptoObj_->decryptChunk(newChunk.chunk);
-    // cout << "decrypted structure data" << endl;
-    // PRINT_BYTE_ARRAY_KEYC(stdout, newChunk.chunk.logicData, newChunk.chunk.logicDataSize);
-    // cryptoObj_->decryptChunk(newChunk.chunk);
-
     bool statusHash = cryptoObj_->generateHash(newChunk.chunk.logicData, newChunk.chunk.logicDataSize, newChunk.chunk.chunkHash);
     if (!statusChunk) {
         cerr << "KeyClient : error encrypt chunk" << endl;
diff --git a/TEDStore/src/client/recvDecode.cpp b/TEDStore/src/client/recvDecode.cpp
@@ -2,25 +2,6 @@
 
 extern Configure config;
 
-void PRINT_BYTE_ARRAY_RECV(
-    FILE* file, void* mem, uint32_t len)
-{
-    if (!mem || !len) {
-        fprintf(file, "\n( null )\n");
-        return;
-    }
-    uint8_t* array = (uint8_t*)mem;
-    fprintf(file, "%u bytes:\n{\n", len);
-    uint32_t i = 0;
-    for (i = 0; i < len - 1; i++) {
-        fprintf(file, "0x%x, ", array[i]);
-        if (i % 8 == 7)
-            fprintf(file, "\n");
-    }
-    fprintf(file, "0x%x ", array[i]);
-    fprintf(file, "\n}\n");
-}
-
 RecvDecode::RecvDecode(string fileName)
 {
     clientID_ = config.getClientID();
@@ -74,15 +55,15 @@ bool RecvDecode::processRecipe(Recipe_t& recipeHead, RecipeList_t& recipeList, u
     memcpy(&respond, respondBuffer, sizeof(NetworkHeadStruct_t));
     if (respond.messageType == ERROR_CLOSE) {
         cerr << "RecvDecode : Server reject download request!" << endl;
-        exit(1);
+        return false;
     }
     if (respond.messageType == ERROR_FILE_NOT_EXIST) {
         cerr << "RecvDecode : Server reject download request, file not exist in server!" << endl;
-        exit(1);
+        return false;
     }
     if (respond.messageType == ERROR_CHUNK_NOT_EXIST) {
         cerr << "RecvDecode : Server reject download request, chunk not exist in server!" << endl;
-        exit(1);
+        return false;
     }
     if (respond.messageType == SUCCESS) {
         uint64_t recipeLength = respond.dataSize;
@@ -100,11 +81,11 @@ bool RecvDecode::processRecipe(Recipe_t& recipeHead, RecipeList_t& recipeList, u
         if (recvSize != respond.dataSize + sizeof(NetworkHeadStruct_t)) {
             cerr << "RecvDecode : recv encrypted file recipe size error" << endl;
         } else {
-            cerr << "RecvDecode : recv encrypted file recipe size =" << respond.dataSize << endl;
+            // cerr << "RecvDecode : recv encrypted file recipe size =" << respond.dataSize << endl;
             u_char clientKey[32];
             memset(clientKey, 1, 32);
             memcpy(&recipeHead, encryptedRecipeBuffer + sizeof(NetworkHeadStruct_t), sizeof(Recipe_t));
-            cerr << "RecvDecode : recv encrypted file recipes done, file size = " << recipeHead.fileRecipeHead.fileSize << endl;
+            // cerr << "RecvDecode : recv encrypted file recipes done, file size = " << recipeHead.fileRecipeHead.fileSize << endl;
             cryptoObj_->decryptWithKey(encryptedRecipeBuffer + sizeof(NetworkHeadStruct_t) + sizeof(Recipe_t), recipeLength - sizeof(Recipe_t), clientKey, decryptedRecipeBuffer);
             u_char* requestChunkList = (u_char*)malloc(sizeof(u_char) * sizeof(RecipeEntry_t) * recipeHead.fileRecipeHead.totalChunkNumber + sizeof(NetworkHeadStruct_t));
             for (uint64_t i = 0; i < recipeHead.fileRecipeHead.totalChunkNumber; i++) {
@@ -129,21 +110,20 @@ bool RecvDecode::processRecipe(Recipe_t& recipeHead, RecipeList_t& recipeList, u
                 cerr << "RecvDecode : storage server closed" << endl;
                 return false;
             } else {
-                cerr << "RecvDecode : process recipe done, send recipe size = " << recipeListSize << " to server" << endl;
-            }
-
-            request.messageType = CLIENT_UPLOAD_DECRYPTED_RECIPE;
-            request.dataSize = recipeHead.fileRecipeHead.totalChunkNumber * sizeof(RecipeEntry_t);
-            sendSize = recipeHead.fileRecipeHead.totalChunkNumber * sizeof(RecipeEntry_t) + sizeof(NetworkHeadStruct_t);
-            memcpy(requestChunkList, &request, sizeof(NetworkHeadStruct_t));
-            if (!socket_.Send(requestChunkList, sendSize)) {
-                free(requestChunkList);
-                cerr << "RecvDecode : storage server closed" << endl;
-                return false;
-            } else {
-                free(requestChunkList);
-                cerr << "RecvDecode : process recipe done, send to server done, send size = " << sendSize << endl;
-                return true;
+                // cerr << "RecvDecode : process recipe done, send recipe size = " << recipeListSize << " to server" << endl;
+                request.messageType = CLIENT_UPLOAD_DECRYPTED_RECIPE;
+                request.dataSize = recipeHead.fileRecipeHead.totalChunkNumber * sizeof(RecipeEntry_t);
+                sendSize = recipeHead.fileRecipeHead.totalChunkNumber * sizeof(RecipeEntry_t) + sizeof(NetworkHeadStruct_t);
+                memcpy(requestChunkList, &request, sizeof(NetworkHeadStruct_t));
+                if (!socket_.Send(requestChunkList, sendSize)) {
+                    free(requestChunkList);
+                    cerr << "RecvDecode : storage server closed" << endl;
+                    return false;
+                } else {
+                    free(requestChunkList);
+                    // cerr << "RecvDecode : process recipe done, send to server done, send size = " << sendSize << endl;
+                    return true;
+                }
             }
         }
     } else {
@@ -176,7 +156,6 @@ void RecvDecode::run()
     u_char requestBuffer[sendSize];
     u_char respondBuffer[NETWORK_MESSAGE_DATA_SIZE];
     int recvSize;
-    uint64_t totalChunkSize = 0;
 
     memcpy(requestBuffer, &request, sizeof(NetworkHeadStruct_t));
     memcpy(requestBuffer + sizeof(NetworkHeadStruct_t), fileNameHash_, FILE_NAME_HASH_SIZE);
@@ -211,16 +190,10 @@ void RecvDecode::run()
                 memcpy(&chunkSize, respondBuffer + sizeof(NetworkHeadStruct_t) + totalRecvSize, sizeof(int));
                 totalRecvSize += sizeof(int);
                 cryptoObj_->decryptChunk(respondBuffer + sizeof(NetworkHeadStruct_t) + totalRecvSize, chunkSize, fileRecipeList_[chunkID].chunkKey, chunkPlaintData);
-                PRINT_BYTE_ARRAY_RECV(stdout, chunkPlaintData, chunkSize);
-                // PRINT_BYTE_ARRAY_RECV(stdout, fileRecipeList_[chunkID].chunkKey, CHUNK_ENCRYPT_KEY_SIZE);
-                // PRINT_BYTE_ARRAY_RECV(stdout, respondBuffer + sizeof(NetworkHeadStruct_t) + totalRecvSize, chunkSize);
-                // cerr << "chunk id = " << chunkID << " chunk size = " << chunkSize << endl;
-                totalChunkSize += chunkSize;
                 RetrieverData_t newData;
                 newData.ID = chunkID;
                 newData.logicDataSize = chunkSize;
                 memcpy(newData.logicData, chunkPlaintData, chunkSize);
-                // PRINT_BYTE_ARRAY_RECV(stdout, chunkPlaintData, chunkSize);
                 if (!insertMQToRetriever(newData)) {
                     cerr << "RecvDecode : Error insert chunk data into retriever" << endl;
                 }
@@ -229,6 +202,6 @@ void RecvDecode::run()
             totalRecvChunks += chunkNumber;
         }
     }
-    cerr << "RecvDecode : download job done, exit now, total chunk size = " << totalChunkSize << endl;
+    cerr << "RecvDecode : download job done, exit now" << endl;
     return;
 }
diff --git a/TEDStore/src/client/retriever.cpp b/TEDStore/src/client/retriever.cpp
@@ -1,24 +1,5 @@
 #include "retriever.hpp"
 
-void PRINT_BYTE_ARRAY_WRITE(
-    FILE* file, void* mem, uint32_t len)
-{
-    if (!mem || !len) {
-        fprintf(file, "\n( null )\n");
-        return;
-    }
-    uint8_t* array = (uint8_t*)mem;
-    fprintf(file, "%u bytes:\n{\n", len);
-    uint32_t i = 0;
-    for (i = 0; i < len - 1; i++) {
-        fprintf(file, "0x%x, ", array[i]);
-        if (i % 8 == 7)
-            fprintf(file, "\n");
-    }
-    fprintf(file, "0x%x ", array[i]);
-    fprintf(file, "\n}\n");
-}
-
 Retriever::Retriever(string fileName, RecvDecode*& recvDecodeObjTemp)
 {
     recvDecodeObj_ = recvDecodeObjTemp;
@@ -40,7 +21,6 @@ void Retriever::recvThread()
     while (totalRecvNumber_ < totalChunkNumber_) {
         if (extractMQFromRecvDecode(newData)) {
             retrieveFile_.write(newData.logicData, newData.logicDataSize);
-            // cerr << newData.logicData << endl;
             totalRecvNumber_++;
         }
     }
diff --git a/TEDStore/src/client/sender.cpp b/TEDStore/src/client/sender.cpp
@@ -10,25 +10,6 @@ struct timeval timeendSenderReadMQ;
 struct timeval timestartSenderRecipe;
 struct timeval timeendSenderRecipe;
 
-void PRINT_BYTE_ARRAY_SENDER(
-    FILE* file, void* mem, uint32_t len)
-{
-    if (!mem || !len) {
-        fprintf(file, "\n( null )\n");
-        return;
-    }
-    uint8_t* array = (uint8_t*)mem;
-    fprintf(file, "%u bytes:\n{\n", len);
-    uint32_t i = 0;
-    for (i = 0; i < len - 1; i++) {
-        fprintf(file, "0x%x, ", array[i]);
-        if (i % 8 == 7)
-            fprintf(file, "\n");
-    }
-    fprintf(file, "0x%x ", array[i]);
-    fprintf(file, "\n}\n");
-}
-
 Sender::Sender()
 {
     inputMQ_ = new messageQueue<Data_t>;
@@ -52,11 +33,8 @@ bool Sender::sendRecipe(Recipe_t request, RecipeList_t recipeList, int& status)
     int totalRecipeNumber = recipeList.size();
     int totalRecipeSize = totalRecipeNumber * sizeof(RecipeEntry_t) + sizeof(Recipe_t);
     u_char* recipeBuffer = (u_char*)malloc(sizeof(u_char) * totalRecipeNumber * sizeof(RecipeEntry_t));
-    PRINT_BYTE_ARRAY_SENDER(stdout, &request.fileRecipeHead.fileNameHash, FILE_NAME_HASH_SIZE);
     for (int i = 0; i < totalRecipeNumber; i++) {
         memcpy(recipeBuffer + i * sizeof(RecipeEntry_t), &recipeList[i], sizeof(RecipeEntry_t));
-        // cerr << "DataSR : recv chunk id = " << recipeList[i].chunkID << ", chunk size = " << recipeList[i].chunkSize << endl;
-        // PRINT_BYTE_ARRAY_SENDER(stdout, recipeList[i].chunkKey, CHUNK_ENCRYPT_KEY_SIZE);
     }
     u_char clientKey[32];
     memset(clientKey, 1, 32);
diff --git a/TEDStore/src/server/dataSR.cpp b/TEDStore/src/server/dataSR.cpp
@@ -62,20 +62,20 @@ void DataSR::run(Socket socket)
                 break;
             }
             case CLIENT_UPLOAD_DECRYPTED_RECIPE: {
-                cerr << "DataSR : current recipe size = " << recipeSize << ", toatl chunk number = " << restoredFileRecipe.fileRecipeHead.totalChunkNumber << endl;
+                // cerr << "DataSR : current recipe size = " << recipeSize << ", toatl chunk number = " << restoredFileRecipe.fileRecipeHead.totalChunkNumber << endl;
                 uint64_t decryptedRecipeListSize = 0;
                 memcpy(&decryptedRecipeListSize, recvBuffer + sizeof(NetworkHeadStruct_t), sizeof(uint64_t));
-                cerr << "DataSR : process recipe list size = " << decryptedRecipeListSize << endl;
+                // cerr << "DataSR : process recipe list size = " << decryptedRecipeListSize << endl;
                 u_char* recvDecryptedRecipeBuffer = (u_char*)malloc(sizeof(u_char) * decryptedRecipeListSize + sizeof(NetworkHeadStruct_t));
                 if (socket.Recv(recvDecryptedRecipeBuffer, recvSize)) {
                     NetworkHeadStruct_t tempHeader;
                     memcpy(&tempHeader, recvDecryptedRecipeBuffer, sizeof(NetworkHeadStruct_t));
-                    cerr << "DataSR : CLIENT_UPLOAD_DECRYPTED_RECIPE, recv message type " << tempHeader.messageType << ", message size = " << tempHeader.dataSize << endl;
+                    // cerr << "DataSR : CLIENT_UPLOAD_DECRYPTED_RECIPE, recv message type " << tempHeader.messageType << ", message size = " << tempHeader.dataSize << endl;
                 } else {
                     cerr << "DataSR : recv decrypted file recipe error " << endl;
                 }
                 int restoreChunkNumber = restoredFileRecipe.fileRecipeHead.totalChunkNumber;
-                cerr << "DataSR : target restore chunk number = " << restoreChunkNumber << endl;
+                // cerr << "DataSR : target restore chunk number = " << restoreChunkNumber << endl;
                 // memcpy(&restoredFileRecipe, recvDecryptedRecipeBuffer + sizeof(NetworkHeadStruct_t), sizeof(Recipe_t));
                 for (int i = 0; i < restoreChunkNumber; i++) {
                     RecipeEntry_t newRecipeEntry;
@@ -90,14 +90,14 @@ void DataSR::run(Socket socket)
             case CLIENT_DOWNLOAD_ENCRYPTED_RECIPE: {
 
                 if (storageObj_->restoreRecipesSize((char*)recvBuffer + sizeof(NetworkHeadStruct_t), recipeSize)) {
-                    cerr << "StorageCore : restore file size = " << recipeSize << endl;
+                    // cerr << "StorageCore : restore file size = " << recipeSize << endl;
                     netBody.messageType = SUCCESS;
                     netBody.dataSize = recipeSize;
                     sendSize = sizeof(NetworkHeadStruct_t);
                     memset(sendBuffer, 0, NETWORK_MESSAGE_DATA_SIZE);
                     memcpy(sendBuffer, &netBody, sizeof(NetworkHeadStruct_t));
                     socket.Send(sendBuffer, sendSize);
-                    cerr << "StorageCore : send recipe size done" << endl;
+                    // cerr << "StorageCore : send recipe size done" << endl;
                     u_char* recipeBuffer = (u_char*)malloc(sizeof(u_char) * recipeSize);
                     storageObj_->restoreRecipes((char*)recvBuffer + sizeof(NetworkHeadStruct_t), recipeBuffer, recipeSize);
                     u_char* sendRecipeBuffer = (u_char*)malloc(sizeof(u_char) * recipeSize + sizeof(NetworkHeadStruct_t));
@@ -106,7 +106,7 @@ void DataSR::run(Socket socket)
                     sendSize = sizeof(NetworkHeadStruct_t) + recipeSize;
                     socket.Send(sendRecipeBuffer, sendSize);
                     memcpy(&restoredFileRecipe, recipeBuffer, sizeof(Recipe_t));
-                    cerr << "StorageCore : send recipe list done, file size = " << restoredFileRecipe.fileRecipeHead.fileSize << ", total chunk number = " << restoredFileRecipe.fileRecipeHead.totalChunkNumber << endl;
+                    // cerr << "StorageCore : send recipe list done, file size = " << restoredFileRecipe.fileRecipeHead.fileSize << ", total chunk number = " << restoredFileRecipe.fileRecipeHead.totalChunkNumber << endl;
                     free(sendRecipeBuffer);
                     free(recipeBuffer);
                 } else {
diff --git a/TEDStore/src/server/storageCore.cpp b/TEDStore/src/server/storageCore.cpp
@@ -8,25 +8,6 @@ extern Configure config;
 extern Database fp2ChunkDB;
 extern Database fileName2metaDB;
 
-void PRINT_BYTE_ARRAY_STORAGE_CORE(
-    FILE* file, void* mem, uint32_t len)
-{
-    if (!mem || !len) {
-        fprintf(file, "\n( null )\n");
-        return;
-    }
-    uint8_t* array = (uint8_t*)mem;
-    fprintf(file, "%u bytes:\n{\n", len);
-    uint32_t i = 0;
-    for (i = 0; i < len - 1; i++) {
-        fprintf(file, "0x%x, ", array[i]);
-        if (i % 8 == 7)
-            fprintf(file, "\n");
-    }
-    fprintf(file, "0x%x ", array[i]);
-    fprintf(file, "\n}\n");
-}
-
 StorageCore::StorageCore()
 {
     RecipeNamePrefix_ = config.getRecipeRootPath();
@@ -82,7 +63,6 @@ bool StorageCore::storeChunks(NetworkHeadStruct_t& networkHead, char* data)
     for (int i = 0; i < chunkNumber; i++) {
         int currentChunkSize;
         string originHash(data + readSize, CHUNK_HASH_SIZE);
-        // cerr << "save chunk hash" << endl;
         readSize += CHUNK_HASH_SIZE;
         memcpy(&currentChunkSize, data + readSize, sizeof(int));
         readSize += sizeof(int);
@@ -126,7 +106,6 @@ bool StorageCore::restoreRecipesSize(char* fileNameHash, uint64_t& recipeSize)
 {
     string recipeName;
     string DBKey(fileNameHash, FILE_NAME_HASH_SIZE);
-    PRINT_BYTE_ARRAY_STORAGE_CORE(stdout, fileNameHash, FILE_NAME_HASH_SIZE);
     if (fileName2metaDB.query(DBKey, recipeName)) {
         ifstream RecipeIn;
         string readRecipeName;
@@ -139,9 +118,6 @@ bool StorageCore::restoreRecipesSize(char* fileNameHash, uint64_t& recipeSize)
             RecipeIn.seekg(0, std::ios::end);
             recipeSize = RecipeIn.tellg();
             RecipeIn.seekg(0, std::ios::beg);
-            // u_char* recipeBuffer = (u_char*)malloc(sizeof(u_char) * recipeSize);
-            // RecipeIn.read((char*)recipeBuffer, recipeSize);
-            // recipeContent = recipeBuffer;
             RecipeIn.close();
             return true;
         }
@@ -156,7 +132,6 @@ bool StorageCore::restoreRecipes(char* fileNameHash, u_char* recipeContent, uint
 {
     string recipeName;
     string DBKey(fileNameHash, FILE_NAME_HASH_SIZE);
-    PRINT_BYTE_ARRAY_STORAGE_CORE(stdout, fileNameHash, FILE_NAME_HASH_SIZE);
     if (fileName2metaDB.query(DBKey, recipeName)) {
         ifstream RecipeIn;
         string readRecipeName;
@@ -187,7 +162,6 @@ bool StorageCore::storeRecipes(char* fileNameHash, u_char* recipeContent, uint64
     string writeRecipeName, buffer, recipeName;
 
     string DBKey(fileNameHash, FILE_NAME_HASH_SIZE);
-    PRINT_BYTE_ARRAY_STORAGE_CORE(stdout, fileNameHash, FILE_NAME_HASH_SIZE);
     if (fileName2metaDB.query(DBKey, recipeName)) {
         cerr << "StorageCore : current file's recipe exist, modify it now" << recipeName << endl;
         writeRecipeName = RecipeNamePrefix_ + recipeName + RecipeNameTail_;
diff --git a/TEDStore/src/util/cryptoPrimitive.cpp b/TEDStore/src/util/cryptoPrimitive.cpp
