Skip to content

Commit 7dea105

Browse files
authored
Merge pull request #459 from fairDataSociety/staging
Staging
2 parents 2713706 + 70c86f5 commit 7dea105

32 files changed

+293
-112
lines changed

.github/workflows/coverge.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
- name: Setup Go
1616
uses: actions/setup-go@v1
1717
with:
18-
go-version: 1.19
18+
go-version: 1.20
1919
- name: Checkout
2020
uses: actions/checkout@v2
2121
with:

.github/workflows/go.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
runs-on: ${{ matrix.os }}
1515
strategy:
1616
matrix:
17-
go: [1.19]
17+
go: [1.20]
1818
os: [ubuntu-latest, macos-latest, windows-latest]
1919
steps:
2020
- name: Setup Go

.github/workflows/release.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
- name: Setup Go
1919
uses: actions/setup-go@v2
2020
with:
21-
go-version: 1.19
21+
go-version: 1.20
2222
- name: Checkout
2323
uses: actions/checkout@v2
2424
with:
@@ -45,7 +45,7 @@ jobs:
4545
- name: Setup Go
4646
uses: actions/setup-go@v2
4747
with:
48-
go-version: 1.19
48+
go-version: 1.20
4949
- name: Checkout
5050
uses: actions/checkout@v2
5151
with:
@@ -85,7 +85,7 @@ jobs:
8585
- name: Setup Go
8686
uses: actions/setup-go@v2
8787
with:
88-
go-version: 1.19
88+
go-version: 1.20
8989
- name: Checkout
9090
uses: actions/checkout@v2
9191
with:

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
GO ?= go
22
GOLANGCI_LINT ?= $$($(GO) env GOPATH)/bin/golangci-lint
3-
GOLANGCI_LINT_VERSION ?= v1.50.0
3+
GOLANGCI_LINT_VERSION ?= v1.52.2
44
GOGOPROTOBUF ?= protoc-gen-gogofaster
55
GOGOPROTOBUF_VERSION ?= v1.3.1
66

cmd/dfs-cli/cmd/filesystem.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,18 +52,16 @@ func isDirectoryPresent(podName, dirNameWithpath string) bool {
5252
return resp.Present
5353
}
5454

55-
func listFileAndDirectories(podName, dirNameWithpath string) {
55+
func listFileAndDirectories(podName, dirNameWithpath string) (*api.ListFileResponse, error) {
5656
args := fmt.Sprintf("podName=%s&dirPath=%s", podName, dirNameWithpath)
5757
data, err := fdfsAPI.getReq(apiDirLs, args)
5858
if err != nil {
59-
fmt.Println("ls failed: ", err)
60-
return
59+
return nil, err
6160
}
6261
var resp api.ListFileResponse
6362
err = json.Unmarshal(data, &resp)
6463
if err != nil {
65-
fmt.Println("dir ls: ", err)
66-
return
64+
return nil, err
6765
}
6866
if resp.Directories == nil && resp.Files == nil {
6967
fmt.Println("empty directory")
@@ -74,6 +72,7 @@ func listFileAndDirectories(podName, dirNameWithpath string) {
7472
for _, entry := range resp.Files {
7573
fmt.Println("<File>: ", entry.Name)
7674
}
75+
return &resp, nil
7776
}
7877

7978
func statFileOrDirectory(podName, statElement string) {
@@ -213,6 +212,7 @@ func uploadFile(fileName, podName, localFileWithPath, podDir, blockSize, compres
213212
args["podName"] = podName
214213
args["dirPath"] = podDir
215214
args["blockSize"] = blockSize
215+
args["overwrite"] = "true"
216216
data, err := fdfsAPI.uploadMultipartFile(apiFileUpload, fileName, fi.Size(), fd, args, "files", compression)
217217
if err != nil {
218218
fmt.Println("upload failed: ", err)

cmd/dfs-cli/cmd/prompt.go

Lines changed: 158 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package cmd
1818

1919
import (
20+
"errors"
2021
"fmt"
2122
"log"
2223
"os"
@@ -228,6 +229,8 @@ var suggestions = []prompt.Suggest{
228229
{Text: "cd", Description: "change path"},
229230
{Text: "download", Description: "download file from dfs to local machine"},
230231
{Text: "upload", Description: "upload file from local machine to dfs"},
232+
{Text: "uploadDir", Description: "upload a dir from local machine to dfs"},
233+
{Text: "downloadDir", Description: "download dir from dfs to local machine"},
231234
{Text: "share", Description: "share file with another user"},
232235
{Text: "receive", Description: "receive a shared file"},
233236
{Text: "exit", Description: "exit dfs-prompt"},
@@ -297,7 +300,6 @@ func executor(in string) {
297300
mnemonic = strings.TrimPrefix(mnemonic, " ")
298301
}
299302
userNew(userName, mnemonic)
300-
currentUser = userName
301303
currentPod = ""
302304
currentDirectory = ""
303305
currentPrompt = getCurrentPrompt()
@@ -750,7 +752,10 @@ func executor(in string) {
750752
if !isPodOpened() {
751753
return
752754
}
753-
listFileAndDirectories(currentPod, currentDirectory)
755+
_, err = listFileAndDirectories(currentPod, currentDirectory)
756+
if err != nil {
757+
fmt.Println("ls failed: ", err)
758+
}
754759
currentPrompt = getCurrentPrompt()
755760
case "mkdir":
756761
if !isPodOpened() {
@@ -797,6 +802,81 @@ func executor(in string) {
797802
}
798803
rmDir(currentPod, dirToRm)
799804
currentPrompt = getCurrentPrompt()
805+
case "uploadDir":
806+
if !isPodOpened() {
807+
return
808+
}
809+
if len(blocks) < 4 {
810+
fmt.Println("invalid command. Missing one or more arguments")
811+
return
812+
}
813+
dirName := blocks[1]
814+
podDir := blocks[2]
815+
if podDir == "." {
816+
podDir = currentDirectory
817+
}
818+
blockSize := blocks[3]
819+
compression := ""
820+
if len(blocks) >= 5 {
821+
compression = blocks[4]
822+
if compression != "snappy" && compression != "gzip" {
823+
fmt.Println("invalid value for \"compression\", should either be \"snappy\" or \"gzip\"")
824+
return
825+
}
826+
}
827+
toUpload, err := findFilesToUpload(dirName)
828+
if err != nil {
829+
fmt.Println("Failed to list files to upload at: ", dirName, err)
830+
return
831+
}
832+
for _, item := range toUpload.filesToUpload {
833+
if toUpload.rootDirectory != item {
834+
itemStats, err := os.Stat(item)
835+
if err != nil {
836+
fmt.Println("Failed to read stats for: ", item, err)
837+
continue
838+
}
839+
fmt.Println("Handling item: ", removeParentDirectory(toUpload.rootDirectory, item))
840+
if itemStats.IsDir() {
841+
dirToMk := filepath.ToSlash(filepath.Join(podDir, removeParentDirectory(toUpload.rootDirectory, item)))
842+
mkdir(currentPod, dirToMk)
843+
} else {
844+
filePath := removeParentDirectory(toUpload.rootDirectory, item)
845+
uploadFile(filepath.Base(filePath), currentPod, item, filepath.ToSlash(filepath.Join(podDir, filepath.Dir(filePath))), blockSize, compression)
846+
}
847+
}
848+
}
849+
currentPrompt = getCurrentPrompt()
850+
case "downloadDir":
851+
if !isPodOpened() {
852+
return
853+
}
854+
if len(blocks) < 3 {
855+
fmt.Println("invalid command. Missing one or more arguments")
856+
return
857+
}
858+
localDir := blocks[1]
859+
dirStat, err := os.Stat(localDir)
860+
if err != nil {
861+
fmt.Println("local path is not a present: ", err)
862+
return
863+
}
864+
865+
if !dirStat.IsDir() {
866+
fmt.Println("local path is not a directory")
867+
return
868+
}
869+
870+
podPath := blocks[2]
871+
if !strings.HasPrefix(podPath, utils.PathSeparator) {
872+
if currentDirectory == utils.PathSeparator {
873+
podPath = currentDirectory + podPath
874+
} else {
875+
podPath = currentDirectory + utils.PathSeparator + podPath
876+
}
877+
}
878+
downloadDir(currentPod, localDir, podPath)
879+
currentPrompt = getCurrentPrompt()
800880
case "upload":
801881
if !isPodOpened() {
802882
return
@@ -988,6 +1068,8 @@ func help() {
9881068
fmt.Println(" - ls ")
9891069
fmt.Println(" - download <destination dir in local fs> <relative path of source file in pod>")
9901070
fmt.Println(" - upload <source file in local fs> <destination directory in pod> <block size (ex: 1Mb, 64Mb)>, <compression (snappy/gzip)>")
1071+
fmt.Println(" - uploadDir <source location in local fs> <destination directory in pod> <block size (ex: 1Mb, 64Mb)>, <compression (snappy/gzip)>")
1072+
fmt.Println(" - downloadDir <destination location in local fs> <source directory in pod>")
9911073
fmt.Println(" - share <file name> - shares a file with another user")
9921074
fmt.Println(" - receive <sharing reference> <pod dir> - receives a file from another user")
9931075
fmt.Println(" - receiveinfo <sharing reference> - shows the received file info before accepting the receive")
@@ -1050,3 +1132,77 @@ func getPassword() (password string) {
10501132
password = strings.TrimSpace(passwd)
10511133
return password
10521134
}
1135+
1136+
type files struct {
1137+
filesToUpload []string
1138+
rootDirectory string
1139+
}
1140+
1141+
func findFilesToUpload(searchPath string) (*files, error) {
1142+
searchResults := []string{}
1143+
rawSearchResults, err := filepath.Glob(searchPath)
1144+
if err != nil {
1145+
return nil, err
1146+
}
1147+
1148+
for _, searchResult := range rawSearchResults {
1149+
searchResults = append(searchResults, searchResult)
1150+
fileStats, err := os.Stat(searchResult)
1151+
if err != nil {
1152+
return nil, err
1153+
}
1154+
1155+
if fileStats.IsDir() {
1156+
toUpload, err := findFilesToUpload(filepath.Join(searchResult, "*"))
1157+
if err == nil {
1158+
searchResults = append(searchResults, toUpload.filesToUpload...)
1159+
}
1160+
}
1161+
}
1162+
1163+
if len(searchResults) == 1 && searchPath == searchResults[0] {
1164+
return &files{
1165+
filesToUpload: searchResults,
1166+
rootDirectory: filepath.Dir(searchResults[0]),
1167+
}, nil
1168+
}
1169+
1170+
return &files{
1171+
filesToUpload: searchResults,
1172+
rootDirectory: searchPath,
1173+
}, nil
1174+
}
1175+
1176+
func removeParentDirectory(parentPath, childPath string) string {
1177+
relativePath, err := filepath.Rel(parentPath, childPath)
1178+
if err != nil {
1179+
panic(err)
1180+
}
1181+
if relativePath == "" {
1182+
return "/"
1183+
}
1184+
if strings.HasPrefix(relativePath, "..") {
1185+
return ""
1186+
}
1187+
return "/" + relativePath
1188+
}
1189+
1190+
func downloadDir(currentPod, localDir, podPath string) {
1191+
stat, err := listFileAndDirectories(currentPod, podPath)
1192+
if err != nil {
1193+
fmt.Println("failed to get contents of dir: ", podPath, err)
1194+
return
1195+
}
1196+
for _, dir := range stat.Directories {
1197+
err := os.Mkdir(filepath.Join(localDir, dir.Name), os.FileMode(dir.Mode))
1198+
if err != nil && !errors.Is(err, os.ErrExist) {
1199+
fmt.Println("failed to create local dir: ", filepath.Join(localDir, dir.Name), err)
1200+
continue
1201+
}
1202+
downloadDir(currentPod, filepath.Join(localDir, dir.Name), filepath.ToSlash(filepath.Join(podPath, dir.Name)))
1203+
}
1204+
for _, file := range stat.Files {
1205+
loalFile := filepath.Join(localDir, file.Name)
1206+
downloadFile(currentPod, loalFile, filepath.ToSlash(filepath.Join(podPath, file.Name)))
1207+
}
1208+
}

cmd/dfs-cli/cmd/user.go

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -53,17 +53,20 @@ func userNew(userName, mnemonic string) {
5353
return
5454
}
5555
if resp.Message == eth.ErrInsufficientBalance.Error() {
56-
fmt.Println(resp.Message)
56+
fmt.Println("Failed to create new user")
57+
fmt.Println("Please fund your account with some eth and try again with the following command")
58+
fmt.Printf(">>> user new %s %s\n", userName, resp.Mnemonic)
5759
fmt.Println("address :", resp.Address)
5860
fmt.Println("=============== Mnemonic ==========================")
5961
fmt.Println(resp.Mnemonic)
6062
fmt.Println("=============== Mnemonic ==========================")
61-
} else {
62-
fmt.Println("user created with address ", resp.Address)
63-
fmt.Println("Please store the 12 words mnemonic safely")
64-
fmt.Println("if you loose that, you cannot recover the data in-case of an emergency.")
65-
fmt.Println("you can also use that mnemonic to access the data in-case this device is lost")
63+
return
6664
}
65+
fmt.Println("user created with address ", resp.Address)
66+
fmt.Println("Please store the 12 words mnemonic safely")
67+
fmt.Println("if you loose that, you cannot recover the data in-case of an emergency.")
68+
fmt.Println("you can also use that mnemonic to access the data in-case this device is lost")
69+
currentUser = userName
6770
}
6871

6972
func userLogin(userName, apiEndpoint string) {

cmd/dfs/cmd/server_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -430,7 +430,7 @@ func TestApis(t *testing.T) {
430430
if err != nil {
431431
t.Fatal(err)
432432
}
433-
err = writer.WriteField("blockSize", "4kb")
433+
err = writer.WriteField("blockSize", "1Mb")
434434
if err != nil {
435435
t.Fatal(err)
436436
}

go.mod

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
module github.com/fairdatasociety/fairOS-dfs
22

3-
go 1.18
3+
go 1.20
44

55
require (
66
github.com/btcsuite/btcd v0.22.3
77
github.com/c-bata/go-prompt v0.2.6
88
github.com/dustin/go-humanize v1.0.1
9-
github.com/ethereum/go-ethereum v1.11.4
10-
github.com/ethersphere/bee v1.12.0
9+
github.com/ethereum/go-ethereum v1.11.5
10+
github.com/ethersphere/bee v1.13.0
1111
github.com/ethersphere/bmt v0.1.4
1212
github.com/fairdatasociety/fairOS-dfs-utils v0.0.0-20221230123929-aec4ed8b854d
1313
github.com/golang/snappy v0.0.4
@@ -26,12 +26,12 @@ require (
2626
github.com/spf13/viper v1.15.0
2727
github.com/stretchr/testify v1.8.2
2828
github.com/swaggo/http-swagger v1.3.4
29-
github.com/swaggo/swag v1.8.10
29+
github.com/swaggo/swag v1.8.12
3030
github.com/tinygrasshopper/bettercsv v0.0.1
3131
github.com/tyler-smith/go-bip39 v1.1.0
3232
github.com/wealdtech/go-ens/v3 v3.5.5
3333
go.uber.org/goleak v1.2.1
34-
golang.org/x/crypto v0.4.0
34+
golang.org/x/crypto v0.7.0
3535
golang.org/x/term v0.6.0
3636
gopkg.in/yaml.v2 v2.4.0
3737
resenje.org/jsonhttp v0.2.0
@@ -54,6 +54,7 @@ require (
5454
github.com/go-stack/stack v1.8.1 // indirect
5555
github.com/google/uuid v1.3.0 // indirect
5656
github.com/hashicorp/hcl v1.0.0 // indirect
57+
github.com/holiman/uint256 v1.2.0 // indirect
5758
github.com/inconshreveable/mousetrap v1.0.1 // indirect
5859
github.com/ipfs/go-cid v0.3.2 // indirect
5960
github.com/josharian/intern v1.0.0 // indirect
@@ -91,12 +92,14 @@ require (
9192
github.com/wealdtech/go-multicodec v1.4.0 // indirect
9293
github.com/yusufpapurcu/wmi v1.2.2 // indirect
9394
go.uber.org/atomic v1.10.0 // indirect
94-
golang.org/x/net v0.7.0 // indirect
95+
golang.org/x/net v0.8.0 // indirect
9596
golang.org/x/sys v0.6.0 // indirect
96-
golang.org/x/text v0.7.0 // indirect
97-
golang.org/x/tools v0.3.0 // indirect
97+
golang.org/x/text v0.8.0 // indirect
98+
golang.org/x/tools v0.7.0 // indirect
9899
gopkg.in/ini.v1 v1.67.0 // indirect
99100
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
100101
gopkg.in/yaml.v3 v3.0.1 // indirect
101102
lukechampine.com/blake3 v1.1.7 // indirect
102103
)
104+
105+
replace github.com/codahale/hdrhistogram => github.com/HdrHistogram/hdrhistogram-go v0.0.0-20200919145931-8dac23c8dac1

0 commit comments

Comments
 (0)