diff --git a/go.mod b/go.mod index 69242e0..612ca69 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/go-playground/validator/v10 v10.11.0 github.com/goccy/go-json v0.9.10 // indirect github.com/google/go-cmp v0.5.6 // indirect + github.com/google/uuid v1.3.0 github.com/mattn/go-colorable v0.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/pelletier/go-toml/v2 v2.0.2 // indirect diff --git a/go.sum b/go.sum index 7e28d93..0c669c9 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= diff --git a/realtime-advanced/go.mod b/realtime-advanced/go.mod index 25f0512..2bf492b 100644 --- a/realtime-advanced/go.mod +++ b/realtime-advanced/go.mod @@ -22,9 +22,9 @@ require ( github.com/pelletier/go-toml/v2 v2.0.1 // indirect github.com/ugorji/go/codec v1.2.7 // indirect golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect - golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect - golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect - golang.org/x/text v0.3.8 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/realtime-advanced/go.sum b/realtime-advanced/go.sum index 4a6c81b..e1cfe97 100644 --- a/realtime-advanced/go.sum +++ b/realtime-advanced/go.sum @@ -61,19 +61,20 @@ github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/upload-file/chunk/client/client.go b/upload-file/chunk/client/client.go new file mode 100644 index 0000000..16ad3e7 --- /dev/null +++ b/upload-file/chunk/client/client.go @@ -0,0 +1,75 @@ +package main + +import ( + "bytes" + "crypto/md5" + "encoding/json" + "fmt" + "io" + "log" + "math" + "net/http" + "os" + "path/filepath" + + "github.com/gin-gonic/examples/upload-file/chunk/model" + "github.com/google/uuid" +) + +func main() { + filePath := "your slice chunk upload file path" + fileName := filepath.Base(filePath) + + fileInfo, err := os.Stat(filePath) + if err != nil { + log.Fatalf("file stat fail: %v\n", err) + return + } + + const chunkSize = 1 << (10 * 2) * 30 + + num := math.Ceil(float64(fileInfo.Size()) / float64(chunkSize)) + + fi, err := os.OpenFile(filePath, os.O_RDONLY, os.ModePerm) + if err != nil { + log.Fatalf("open file fail: %v\n", err) + return + } + + fileId := uuid.NewString() + + fileKeys := make([]string, 0) + for i := 1; i <= int(num); i++ { + file := make([]byte, chunkSize) + fi.Seek((int64(i)-1)*chunkSize, 0) + if len(file) > int(fileInfo.Size()-(int64(i)-1)*chunkSize) { + file = make([]byte, fileInfo.Size()-(int64(i)-1)*chunkSize) + } + fi.Read(file) + + key := fmt.Sprintf("%x", md5.Sum(file)) + + fileKeys = append(fileKeys, key) + + req := model.ChunkFileRequest{ + FileId: fileId, + FileName: fileName, + FileIndex: i, + FileCount: int(num), + FileKey: key, + FileKeys: fileKeys, + File: file, + } + body, _ := json.Marshal(req) + + res, err := http.Post("http://127.0.0.1:8080/chunkUploadFile", "application/json", bytes.NewBuffer(body)) + + if err != nil { + log.Fatalf("http post fail: %v", err) + return + } + defer res.Body.Close() + msg, _ := io.ReadAll(res.Body) + fmt.Println(string(msg)) + } +} diff --git a/upload-file/chunk/model/chunk_model.go b/upload-file/chunk/model/chunk_model.go new file mode 100644 index 0000000..8293fbb --- /dev/null +++ b/upload-file/chunk/model/chunk_model.go @@ -0,0 +1,127 @@ +package model + +import ( + "crypto/md5" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + + "github.com/gin-gonic/gin" +) + +type ChunkFileRequest struct { + FileId string `json:"fileId"` // client create uuid + FileName string `json:"fileName"` // file name + FileIndex int `json:"fileIndex"` // file index + FileCount int `json:"fileCount"` // file slice size + FileKeys []string `json:"fileKeys"` // file slice all key md5 (accumulation file key if fileIndex == fileCount then merge after verification) + FileKey string `json:"fileKey"` // file now key to md5 - if server read the slice to md5 eq key not eq then fail + File []byte `json:"file"` // now file +} + +func (cf *ChunkFileRequest) BindingForm(ctx *gin.Context) error { + if err := ctx.ShouldBind(cf); err != nil { + return err + } + + return cf.md5() +} + +func (cf *ChunkFileRequest) md5() error { + hash := fmt.Sprintf("%x", md5.Sum(cf.File)) + if hash != cf.FileKey { + return errors.New("current file slice key error") + } + return nil +} + +func (cf *ChunkFileRequest) SaveUploadedFile(tempPath, path string) (string, error) { + tempFolder := filepath.Join(tempPath, cf.FileId) + + _, err := os.Stat(tempFolder) + if os.IsNotExist(err) { + err := os.MkdirAll(tempFolder, os.ModePerm) + if err != nil { + return "", err + } + } + + out, err := os.Create(filepath.Join(tempFolder, cf.FileKey)) + if err != nil { + return "", err + } + defer out.Close() + if _, err := out.Write(cf.File); err != nil { + return "", err + } + + fmt.Println(cf.FileIndex, cf.FileCount) + if cf.FileIndex != cf.FileCount { + return "", nil + } + for _, fileKey := range cf.FileKeys { + tempFile := filepath.Join(tempFolder, fileKey) + if _, err := os.Stat(tempFile); err != nil { + return "", errors.New("file " + fileKey + " is emtpy") + } + } + + base := filepath.Dir(path) + if _, err := os.Stat(base); err != nil { + if os.IsNotExist(err) { + err := os.MkdirAll(base, os.ModePerm) + if err != nil { + return "", err + } + } + } + + file, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0664) + if err != nil { + return "", err + } + + defer file.Close() + + for _, fileKey := range cf.FileKeys { + tempFile := filepath.Join(tempFolder, fileKey) + bt, err := os.ReadFile(tempFile) + if err != nil { + return "", err + } + file.Write(bt) + } + + return tempFolder, nil +} + +// request method is json +// param: fileId +// param: fileName +// param: fileIndex the file slice index +// param: fileCount the file slice size +// param: fileKeys the file slice all file key md5 (accumulation file key if fileIndex == fileCount then merge after verification) +// param: fileKey now file slice key md5 +// param: file now slice file +func ChunkUploadFile(ctx *gin.Context) { + var cf ChunkFileRequest + + if err := cf.BindingForm(ctx); err != nil { + ctx.JSON(http.StatusBadRequest, gin.H{"code": "400", "msg": "bad file param", "err": err.Error()}) + return + } + + tempFolder, err := cf.SaveUploadedFile("./temp", "./uploads/"+cf.FileName) + if err != nil { + ctx.JSON(http.StatusServiceUnavailable, gin.H{"code": "503", "msg": "bad save upload file", "err": err.Error()}) + return + } + ctx.JSON(http.StatusOK, gin.H{"code": "200", "msg": "success"}) + if tempFolder != "" { + defer func(tempFolder string) { + os.RemoveAll(tempFolder) + }(tempFolder) + } +} diff --git a/upload-file/chunk/server/server.go b/upload-file/chunk/server/server.go new file mode 100644 index 0000000..39c2583 --- /dev/null +++ b/upload-file/chunk/server/server.go @@ -0,0 +1,12 @@ +package main + +import ( + "github.com/gin-gonic/examples/upload-file/chunk/model" + "github.com/gin-gonic/gin" +) + +func main() { + g := gin.Default() + g.POST("/chunkUploadFile", model.ChunkUploadFile) + g.Run(":8080") +}