support nginx upload module proxy_pass method

support configuration of HASH algorithm type

Author: xengine-qyt

APIList.txt

@@ -0,0 +1,12 @@
+只支持POST,业务处理端口
+
+查询文件列表:/api/query/file
+
+{
+    "lpszTimeStart":"开始时间,可NULL",
+    "lpszTimeEnd":"结束时间,可NULL",
+    "lpszFileName":"文件名,可NULL",
+    "lpszFileHash":"文件HASH,可NULL"
+}
+
+NGINX上传代理接口:/api/event/upfile

README.en.md

@@ -12,7 +12,7 @@ This repository has a development and master branch. If you want to use it, plea
 
 feature list:  
 1. support file http upload and download(use put and get method)  
-2. support private protocol transmission data(planning)  
+2. support http api notify of event and management api interface(planning)  
 3. support encrypt data transimission(planning)  
 4. support mysql record infomation.  
 5. support load banace(planning)  
@@ -21,6 +21,9 @@ feature list:
 8. supprot speeds contral  
 9. supprot task start and end proxy forwarding(planning)  
 10.support p2p,bt and many more(planning)  
+11.support second pass  
+12.support nginx download proxy_pass  
+13.support nginx upload module proxy_pass  
 
 ## install
 
@@ -30,7 +33,7 @@ XEngine need V7.13 or above
 vcpkg need 2021.05.11 or above  
 
 #### Windows
-use vs2019 open and complie  
+use vs2019 x86 open and complie  
 You need to configure the environment in the following way, otherwise you may need to set the library directory in the project yourself  
 
 ##### install Dependent library
@@ -73,6 +76,17 @@ make FLAGS=CleanAll clear
 5.  run
 6.  use curl or postman test upload and download
 
+## api list
+POST Method used as api server  
+it is used as manage service.api format use to url,such as:POST /api/query/file  
+The three-segment format is fixed,first api is a fixed,second api of query is a type,third api of file is a name  
+support api list reference:apilist.txt  
+
+## second pass
+The Second pass is not realized by the server, it is by the client  
+upload file second pass is first check the HASH file is on the server, if has file on the server, it will directly prompt the upload is successful.  
+The realization of downloading second transmission is to first query the local file save path through HASH, and download it directly if it exists.
+
 ## directory struct
 - XEngine_Docment  docment directory
 - XEngine_Release  install directory

README.md

@@ -1,25 +1,28 @@
 # XEngine_Storage
 
 ## 介绍
-c c++ 存储服务 c c++ 文件存储服务
+c c++ 存储服务 c c++ 文件存储服务  
 c c++ file storage service  
 这是一个上传下载服务API网关,不是文件同步服务,也不是云存储.  
-这个服务的主要目的是为了给有上传下载服务业务需求的用户使用(比如需要上传下载服务器的开发人员).  
+这个服务的主要目的是为了给有上传下载服务业务需求的用户使用(比如需要开发上传下载服务的开发人员).  
 
 ## 软件特性
 基于libXEngine开发并实现的一套简洁高性能跨平台网络存储服务  
 本仓库有开发和主分支,如果要使用,请使用master分支下的代码  
 软件特性:  
 1. 支持HTTP协议上传和下载(采用PUT和GET)
-2. 支持二进制私有协议传送数据(规划中)
+2. 支持HTTP API接口事件通知与管理(规划中)
 3. 支持加密传输(规划中)
 4. 采用MYSQL记录信息
 5. 支持分布式(规划中)
 6. 支持HTTP和私有验证(规划中)
 7. 支持详尽的日志和配置信息
 8. 支持传输速率控制
-9. 支持任务开始结束代理转发(规划中)
-10.支持P2P,BT,数据分发(规划中)
+9. 支持任务开始结束代理转发(规划中)  
+10.支持P2P,BT,数据分发(规划中)  
+11.支持秒传  
+12.支持NGINX 下载透传代理(规划中)  
+13.支持NGINX UPLOAD MODULE上传代理  
 
 ## 安装教程
 
@@ -29,7 +32,7 @@ XEngine版本需要V7.13或者以上版本
 vcpkg 需要2021.05.11以上版本  
 
 #### Windows
-使用VS2019 打开并且编译
+使用VS2019 x86打开并且编译
 你需要按照下面的方式配置环境,不然你可能需要自己在项目中设置库目录
 ##### 三方库环境
 需要使用VCPKG安装环境.代码地址:https://github.com/microsoft/vcpkg  
@@ -75,6 +78,17 @@ make FLAGS=CleanAll 清理编译
 - XEngine_Release  安装目录结构  
 - XEngine_Source   源代目录结构  
 
+## API列表
+POST方法在本服务中用作于API.  
+API用于管理服务器.API列表格式使用URL来识别,比如,POST /api/query/file  
+三段格式是固定的,第一个api固定值,第二个query表示API类型,第三个file表示api名  
+支持的API列表参考:apilist.txt  
+
+## 秒传实现  
+秒传的实现不是靠服务器实现的,而是靠客户端实现的.  
+上传秒传的实现是先通过HASH查询文件是否在服务器,如果存在就不上传直接提示客户端上传成功.  
+下载秒传的实现是先通过HASH查询本地文件保存路径,如果存在就直接下载完成.  
+
 ## 参与贡献
 
 1.  Fork 本仓库

XEngine_Release/XEngine_Config/XEngine_Config.json

@@ -32,6 +32,7 @@
   },
   "XStorage": {
     "nUseMode": 1,
+    "nHashMode": 2,
     "tszHttpAddr": "http://192.168.1.4",
     "tszNginAddr": "http://192.168.1.4:5010",
     "tszFileDir": "./XEngine_File"
@@ -42,6 +43,7 @@
   },
   "XVer": {
     "StorageVersion": [
+      "1.2.0.1001 Build20210618",
       "1.1.0.1001 Build20210611",
       "1.0.0.1001 Build20210501"
     ]

XEngine_Source/StorageModule_Config/Config_Define.h

@@ -54,6 +54,7 @@ typedef struct tag_XEngine_ServerConfig
     struct  
 	{
         int nUseMode;
+        int nHashMode;
         TCHAR tszHttpAddr[MAX_PATH];
         TCHAR tszNginAddr[MAX_PATH];
 		TCHAR tszFileDir[MAX_PATH];

XEngine_Source/StorageModule_Config/Config_Json/Config_Json.cpp

@@ -119,14 +119,15 @@ BOOL CConfig_Json::Config_Json_File(LPCTSTR lpszConfigFile,XENGINE_SERVERCONFIG
     _tcscpy(pSt_ServerConfig->st_XSql.tszSQLUser,st_JsonXSql["SQLUser"].asCString());
     _tcscpy(pSt_ServerConfig->st_XSql.tszSQLPass,st_JsonXSql["SQLPass"].asCString());
 
-    if (st_JsonRoot["XStorage"].empty() || (4 != st_JsonRoot["XStorage"].size()))
+    if (st_JsonRoot["XStorage"].empty() || (5 != st_JsonRoot["XStorage"].size()))
     {
         Config_IsErrorOccur = TRUE;
         Config_dwErrorCode = ERROR_XENGINE_BLOGIC_CONFIG_JSON_XSTORAGE;
         return FALSE;
     }
     Json::Value st_JsonXStorage = st_JsonRoot["XStorage"];
 	pSt_ServerConfig->st_XStorage.nUseMode = st_JsonXStorage["nUseMode"].asInt();
+    pSt_ServerConfig->st_XStorage.nHashMode = st_JsonXStorage["nHashMode"].asInt();
 	_tcscpy(pSt_ServerConfig->st_XStorage.tszHttpAddr, st_JsonXStorage["tszHttpAddr"].asCString());
 	_tcscpy(pSt_ServerConfig->st_XStorage.tszNginAddr, st_JsonXStorage["tszNginAddr"].asCString());
     _tcscpy(pSt_ServerConfig->st_XStorage.tszFileDir, st_JsonXStorage["tszFileDir"].asCString());

XEngine_Source/XEngine_StorageApp/Makefile

@@ -5,7 +5,7 @@ LOADBIN = -L /usr/local/lib/XEngine_Release/XEngine_BaseLib -L /usr/local/lib/XE
 LIB = -lXEngine_BaseLib -lXEngine_Algorithm -lXEngine_Core -lXEngine_ManagePool -lXEngine_OPenSsl -lHelpComponents_XLog -lXEngine_ProcSdk -lXEngine_SystemApi -lRfcComponents_HttpServer -lXStorage_SQLPacket -lXStorage_Protocol -lStorageModule_Config -lStorageModule_Session
 LIBEX = -static-libgcc -ldl -lrt -lpthread
 LOADSO = -Wl,-rpath=./,--disable-new-dtags
-LIBINCLUDE = StorageApp_Config.o StorageApp_Download.o StorageApp_Network.o StorageApp_Task.o StorageApp_UPLoader.o XEngine_StorageApp.o
+LIBINCLUDE = StorageApp_Config.o StorageApp_Download.o StorageApp_Network.o StorageApp_Center.o StorageApp_UPLoader.o XEngine_StorageApp.o
 
 ifeq ($(RELEASE),1)
 FLAGS = -c -O2 
@@ -30,8 +30,8 @@ StorageApp_Download.o:./StorageApp_Download.cpp
 	$(CC) $(DEBUG) $(FLAGS) $(UNICODE) ./StorageApp_Download.cpp
 StorageApp_Network.o:./StorageApp_Network.cpp
 	$(CC) $(DEBUG) $(FLAGS) $(UNICODE) ./StorageApp_Network.cpp 
-StorageApp_Task.o:./StorageApp_Task.cpp
-	$(CC) $(DEBUG) $(FLAGS) $(UNICODE) ./StorageApp_Task.cpp
+StorageApp_Center.o:./StorageApp_Center.cpp
+	$(CC) $(DEBUG) $(FLAGS) $(UNICODE) ./StorageApp_Center.cpp
 StorageApp_UPLoader.o:./StorageApp_UPLoader.cpp
 	$(CC) $(DEBUG) $(FLAGS) $(UNICODE) ./StorageApp_UPLoader.cpp
 XEngine_StorageApp.o:./XEngine_StorageApp.cpp             

XEngine_Source/XEngine_StorageApp/StorageApp_Center.cpp

@@ -24,10 +24,12 @@ XHTHREAD CALLBACK XEngine_Center_HTTPThread(LPVOID lParam)
 				for (int j = 0; j < ppSt_PKTClient[i]->nPktCount; j++)
 				{
 					int nMsgLen = 10240;
+					int nHdrCount = 0;
+					CHAR** ppszListHdr = NULL;
 					//获得指定上传客户端触发信息
-					if (RfcComponents_HttpServer_GetClientEx(xhCenterHttp, ppSt_PKTClient[i]->tszClientAddr, tszMsgBuffer, &nMsgLen, &st_HTTPParam))
+					if (RfcComponents_HttpServer_GetClientEx(xhCenterHttp, ppSt_PKTClient[i]->tszClientAddr, tszMsgBuffer, &nMsgLen, &st_HTTPParam, &ppszListHdr, &nHdrCount))
 					{
-						XEngine_Task_HttpCenter(ppSt_PKTClient[i]->tszClientAddr, tszMsgBuffer, nMsgLen, &st_HTTPParam);
+						XEngine_Task_HttpCenter(ppSt_PKTClient[i]->tszClientAddr, tszMsgBuffer, nMsgLen, &st_HTTPParam, ppszListHdr, nHdrCount);
 					}
 				}
 			}
@@ -64,7 +66,7 @@ BOOL XEngine_Task_HttpCenter_APIList(LPCTSTR lpszUrlName, TCHAR* ptszAPIVersion,
 	_tcscpy(ptszAPIName, ptszTokStr);
 	return TRUE;
 }
-BOOL XEngine_Task_HttpCenter(LPCTSTR lpszClientAddr, LPCTSTR lpszMsgBuffer, int nMsgLen, RFCCOMPONENTS_HTTP_REQPARAM* pSt_HTTPParam)
+BOOL XEngine_Task_HttpCenter(LPCTSTR lpszClientAddr, LPCTSTR lpszMsgBuffer, int nMsgLen, RFCCOMPONENTS_HTTP_REQPARAM* pSt_HTTPParam, TCHAR** pptszListHdr, int nHdrCount)
 {
 	int nSDLen = 2048;
 	TCHAR tszSDBuffer[2048];
@@ -101,11 +103,13 @@ BOOL XEngine_Task_HttpCenter(LPCTSTR lpszClientAddr, LPCTSTR lpszMsgBuffer, int
 		return FALSE;
 	}
 
+	LPCTSTR lpszEvent = _T("Event");
 	LPCTSTR lpszQuery = _T("query");
+	LPCTSTR lpszEventUPFile = _T("UPFile");
 	LPCTSTR lpszQueryFile = _T("file");
-	if (0 == _tcsncmp(lpszQuery, tszAPIMethod, _tcslen(lpszQuery)))
+	if (0 == _tcsnicmp(lpszQuery, tszAPIMethod, _tcslen(lpszQuery)))
 	{
-		if (0 == _tcsncmp(lpszQueryFile, tszAPIName, _tcslen(lpszQueryFile)))
+		if (0 == _tcsnicmp(lpszQueryFile, tszAPIName, _tcslen(lpszQueryFile)))
 		{
 			int nMsgLen = 10240;
 			TCHAR tszFileName[MAX_PATH];
@@ -135,5 +139,65 @@ BOOL XEngine_Task_HttpCenter(LPCTSTR lpszClientAddr, LPCTSTR lpszMsgBuffer, int
 			BaseLib_OperatorMemory_Free((XPPPMEM)&ppSt_ListFile, nListCount);
 		}
 	}
+	else if (0 == _tcsnicmp(lpszEvent, tszAPIMethod, _tcslen(lpszEvent)))
+	{
+		if (0 == _tcsnicmp(lpszEventUPFile, tszAPIName, _tcslen(lpszEventUPFile)))
+		{
+			LPCTSTR lpszContentType = _T("Content-Type");
+			LPCTSTR lpszBoundaryStr = _T("boundary=");
+			TCHAR tszContentStr[MAX_PATH];
+			TCHAR tszBoundarTmp[MAX_PATH];
+			TCHAR tszBoundarStr[MAX_PATH];
+			TCHAR tszFileDir[MAX_PATH];
+			XSTORAGECORE_DBFILE st_DBFile;
+
+			memset(tszContentStr, '\0', MAX_PATH);
+			memset(tszBoundarTmp, '\0', MAX_PATH);
+			memset(tszBoundarStr, '\0', MAX_PATH);
+			memset(tszContentStr, '\0', MAX_PATH);
+			memset(tszFileDir, '\0', MAX_PATH);
+			memset(&st_DBFile, '\0', sizeof(XSTORAGECORE_DBFILE));
+
+			if (!RfcComponents_HttpHelp_GetField(&pptszListHdr, nHdrCount, lpszContentType, tszContentStr))
+			{
+				st_HDRParam.bIsClose = TRUE;
+				st_HDRParam.nHttpCode = 400;
+
+				RfcComponents_HttpServer_SendMsgEx(xhCenterHttp, tszSDBuffer, &nSDLen, &st_HDRParam);
+				XEngine_Net_SendMsg(lpszClientAddr, tszSDBuffer, nSDLen, STORAGE_NETTYPE_HTTPCENTER);
+				XLOG_PRINT(xhLog, XENGINE_HELPCOMPONENTS_XLOG_IN_LOGLEVEL_ERROR, _T("业务客户端:%s,请求的HTTP头内容不正确,没有类型字段"), lpszClientAddr);
+				return FALSE;
+			}
+			if (!BaseLib_OperatorString_FromStrGetKeyValue(tszContentStr, lpszBoundaryStr, NULL, tszBoundarTmp))
+			{
+				st_HDRParam.bIsClose = TRUE;
+				st_HDRParam.nHttpCode = 400;
+
+				RfcComponents_HttpServer_SendMsgEx(xhCenterHttp, tszSDBuffer, &nSDLen, &st_HDRParam);
+				XEngine_Net_SendMsg(lpszClientAddr, tszSDBuffer, nSDLen, STORAGE_NETTYPE_HTTPCENTER);
+				XLOG_PRINT(xhLog, XENGINE_HELPCOMPONENTS_XLOG_IN_LOGLEVEL_ERROR, _T("业务客户端:%s,请求的HTTP头内容不正确,没有Boundary值"), lpszClientAddr);
+				return FALSE;
+			}
+			_stprintf(tszBoundarStr, _T("--%s\r\n"), tszBoundarTmp);
+			
+			XStorageProtocol_Core_REQUPEvent(lpszMsgBuffer, tszBoundarStr, st_DBFile.st_ProtocolFile.tszFileName, tszFileDir, st_DBFile.st_ProtocolFile.tszFileHash, &st_DBFile.st_ProtocolFile.nFileSize);
+			_tcscpy(st_DBFile.st_ProtocolFile.tszFilePath, st_ServiceCfg.st_XStorage.tszFileDir);
+
+			if (XStorageSQL_File_FileInsert(&st_DBFile))
+			{
+				st_HDRParam.nHttpCode = 200;
+				RfcComponents_HttpServer_SendMsgEx(xhUPHttp, tszSDBuffer, &nSDLen, &st_HDRParam);
+				XEngine_Net_SendMsg(lpszClientAddr, tszSDBuffer, nSDLen, STORAGE_NETTYPE_HTTPCENTER);
+				XLOG_PRINT(xhLog, XENGINE_HELPCOMPONENTS_XLOG_IN_LOGLEVEL_INFO, _T("业务客户端:%s,处理NGINX代理上传文件成功,文件名:%s,大小:%lld"), lpszClientAddr, tszFileDir, st_DBFile.st_ProtocolFile.nFileSize);
+			}
+			else
+			{
+				st_HDRParam.nHttpCode = 403;
+				RfcComponents_HttpServer_SendMsgEx(xhUPHttp, tszSDBuffer, &nSDLen, &st_HDRParam);
+				XEngine_Net_SendMsg(lpszClientAddr, tszSDBuffer, nSDLen, STORAGE_NETTYPE_HTTPCENTER);
+				XLOG_PRINT(xhLog, XENGINE_HELPCOMPONENTS_XLOG_IN_LOGLEVEL_ERROR, _T("业务客户端:%s,处理NGINX代理上传文件失败,插入数据库失败:%s,错误:%lX"), lpszClientAddr, tszFileDir, XStorageDB_GetLastError());
+			}
+		}
+	}
 	return TRUE;
 }

XEngine_Source/XEngine_StorageApp/StorageApp_Center.h

@@ -2,4 +2,4 @@
 
 XHTHREAD CALLBACK XEngine_Center_HTTPThread(LPVOID lParam);
 BOOL XEngine_Task_HttpCenter_APIList(LPCTSTR lpszUrlName, TCHAR* ptszAPIVersion, TCHAR* ptszAPIMethod, TCHAR* ptszAPIName);
-BOOL XEngine_Task_HttpCenter(LPCTSTR lpszClientAddr, LPCTSTR lpszMsgBuffer, int nMsgLen, RFCCOMPONENTS_HTTP_REQPARAM* pSt_HTTPParam);
+BOOL XEngine_Task_HttpCenter(LPCTSTR lpszClientAddr, LPCTSTR lpszMsgBuffer, int nMsgLen, RFCCOMPONENTS_HTTP_REQPARAM* pSt_HTTPParam, TCHAR** pptszListHdr, int nHdrCount);

XEngine_Source/XEngine_StorageApp/StorageApp_Hdr.h

@@ -68,7 +68,6 @@ extern XHANDLE xhCenterHttp;
 extern XENGINE_SERVERCONFIG st_ServiceCfg;
 
 #include "StorageApp_Network.h"
-#include "StorageApp_Task.h"
 #include "StorageApp_Config.h"
 #include "StorageApp_Download.h"
 #include "StorageApp_UPLoader.h"