Skip to content

Commit 6b2dc97

Browse files
committed
Merge pull request #29 from ipfs/migration-stress
Migration stress
2 parents 7c25684 + a61acbf commit 6b2dc97

File tree

12 files changed

+754
-2
lines changed

12 files changed

+754
-2
lines changed

sharness/Makefile

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,14 @@ SHARNESSDIR = sharness
1111
AGGREGATE = $(LIBDIR)/$(SHARNESSDIR)/aggregate-results.sh
1212

1313
# Binaries generated
14-
BINS = bin/fs-repo-migrations bin/ipfs-0-to-1 bin/ipfs-1-to-2 bin/ipfs-2-to-3 bin/ipfs-update
14+
BINS =
15+
BINS += bin/fs-repo-migrations
16+
BINS += bin/ipfs-0-to-1
17+
BINS += bin/ipfs-1-to-2
18+
BINS += bin/ipfs-2-to-3
19+
BINS += bin/ipfs-update
20+
BINS += bin/random-files
21+
BINS += bin/go-sleep
1522

1623
# Source files location
1724
FS_REPO_SRC = ../
@@ -58,6 +65,16 @@ bin/ipfs-update: $(call find_go_files, $(IPFS_UPDATE_SRC)) BUILD-OPTIONS
5865
@echo "*** installing $@ ***"
5966
go build $(GOFLAGS) -o $@ $(IPFS_UPDATE_SRC)
6067

68+
bin/random-files:
69+
@echo "*** installing $@ ***"
70+
go get github.com/jbenet/go-random-files/random-files
71+
go build $(GOFLAGS) -o $@ github.com/jbenet/go-random-files/random-files
72+
73+
bin/go-sleep:
74+
@echo "*** installing $@ ***"
75+
go get github.com/chriscool/go-sleep
76+
go build $(GOFLAGS) -o $@ github.com/chriscool/go-sleep
77+
6178
bin/ipfs-0-to-1: $(call find_go_files, $(IPFS_0_TO_1_SRC)) BUILD-OPTIONS
6279
@echo "*** installing $@ ***"
6380
go build $(GOFLAGS) -o $@ $(IPFS_0_TO_1_SRC)
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
The MIT License (MIT)
2+
3+
Copyright (c) 2014 Juan Batiz-Benet
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in
13+
all copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21+
THE SOFTWARE.
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
build:
2+
cd random-files && go build
3+
4+
install:
5+
cd random-files && go install
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# go-random-files - create random fs hierarchies
2+
3+
Useful for testing filesystems.
4+
5+
6+
```
7+
# library
8+
go get -u github.com/jbenet/go-random-files
9+
10+
# binary
11+
go get -u github.com/jbenet/go-random-files/random-files
12+
```
13+
14+
### Library godoc: https://godoc.org/github.com/jbenet/go-random-files
15+
16+
# random-files - create random fs hierarchies
17+
18+
See more about the binary at: github.com/jbenet/go-random-files/tree/master/random-files
19+
20+
Useful for testing filesystems.
21+
22+
## Install
23+
24+
```
25+
go get -u github.com/jbenet/go-random-files/random-files
26+
```
27+
28+
## Usage
29+
30+
```sh
31+
> random-files --help
32+
usage: random-files [options] <path>...
33+
Write a random filesystem hierarchy to each <path>
34+
35+
Options:
36+
-alphabet="easy": alphabet for filenames {easy, hard}
37+
-depth=2: fanout depth - how deep the hierarchy goes
38+
-dirs=5: fanout dirs - number of dirs per dir (or max)
39+
-files=10: fanout files - number of files per dir (or max
40+
-filesize=4096: filesize - how big to make each file (or max)
41+
-q=false: quiet output
42+
-random-crypto=false: use cryptographic randomness for files
43+
-random-fanout=false: randomize fanout numbers
44+
-random-size=true: randomize filesize
45+
-seed=0: random seed - 0 for current time
46+
```
47+
48+
### Examples
49+
50+
```sh
51+
> random-files --depth=2 --files=3 foo
52+
foo/h20uo3jrpihb
53+
foo/x6tef1
54+
foo/jh0c2vdci
55+
foo/fden012m368
56+
foo/fden012m368/p6n0chy4kg
57+
foo/fden012m368/h92_
58+
foo/fden012m368/kvjiya98p3
59+
foo/e_i6hwav1tb
60+
foo/e_i6hwav1tb/oj0-a
61+
foo/e_i6hwav1tb/1-pfgvim
62+
foo/e_i6hwav1tb/s_unf
63+
foo/bgvy8x-_hsm
64+
foo/bgvy8x-_hsm/98zcoz-9ng
65+
foo/bgvy8x-_hsm/j0see3qv
66+
foo/bgvy8x-_hsm/qntuf0r
67+
foo/6zjkw3ejm2awwt
68+
foo/6zjkw3ejm2awwt/iba52dh1lhnewh
69+
foo/6zjkw3ejm2awwt/n1bwcv5zpe
70+
foo/6zjkw3ejm2awwt/o8k89cc
71+
foo/efp_6
72+
foo/efp_6/qfap2
73+
foo/efp_6/v_kl_wlefsaa
74+
foo/efp_6/r7sdbph
75+
```
76+
77+
It made:
78+
79+
```
80+
> tree foo
81+
foo
82+
├── 6zjkw3ejm2awwt
83+
│   ├── iba52dh1lhnewh
84+
│   ├── n1bwcv5zpe
85+
│   └── o8k89cc
86+
├── bgvy8x-_hsm
87+
│   ├── 98zcoz-9ng
88+
│   ├── j0see3qv
89+
│   └── qntuf0r
90+
├── e_i6hwav1tb
91+
│   ├── 1-pfgvim
92+
│   ├── oj0-a
93+
│   └── s_unf
94+
├── efp_6
95+
│   ├── qfap2
96+
│   ├── r7sdbph
97+
│   └── v_kl_wlefsaa
98+
├── fden012m368
99+
│   ├── h92_
100+
│   ├── kvjiya98p3
101+
│   └── p6n0chy4kg
102+
├── h20uo3jrpihb
103+
├── jh0c2vdci
104+
└── x6tef1
105+
106+
5 directories, 18 files
107+
```
108+
Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
package randomfiles
2+
3+
import (
4+
"fmt"
5+
"io"
6+
"math/rand"
7+
"os"
8+
"path"
9+
)
10+
11+
type Options struct {
12+
Out io.Writer // output progress
13+
Source io.Reader // randomness source
14+
15+
FileSize int // the size per file.
16+
Alphabet []rune // for filenames
17+
18+
FanoutDepth int // how deep the hierarchy goes
19+
FanoutFiles int // how many files per dir
20+
FanoutDirs int // how many dirs per dir
21+
22+
RandomSeed int64 // use a random seed. if 0, use a random seed
23+
RandomSize bool // randomize file sizes
24+
RandomFanout bool // randomize fanout numbers
25+
}
26+
27+
func WriteRandomFiles(root string, depth int, opts *Options) error {
28+
29+
numfiles := opts.FanoutFiles
30+
if opts.RandomFanout {
31+
numfiles = rand.Intn(numfiles) + 1
32+
}
33+
34+
for i := 0; i < numfiles; i++ {
35+
if err := WriteRandomFile(root, opts); err != nil {
36+
return err
37+
}
38+
}
39+
40+
if depth+1 <= opts.FanoutDepth {
41+
numdirs := opts.FanoutDirs
42+
if opts.RandomFanout {
43+
numdirs = rand.Intn(numdirs) + 1
44+
}
45+
46+
for i := 0; i < numdirs; i++ {
47+
if err := WriteRandomDir(root, depth+1, opts); err != nil {
48+
return err
49+
}
50+
}
51+
}
52+
53+
return nil
54+
}
55+
56+
var FilenameSize = 16
57+
var RunesEasy = []rune("abcdefghijklmnopqrstuvwxyz01234567890-_")
58+
var RunesHard = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890!@#$%^&*()-_+= ;.,<>'\"[]{}() ")
59+
60+
func RandomFilename(length int, alphabet []rune) string {
61+
b := make([]rune, length)
62+
for i := range b {
63+
b[i] = alphabet[rand.Intn(len(alphabet))]
64+
}
65+
return string(b)
66+
}
67+
68+
func WriteRandomFile(root string, opts *Options) error {
69+
filesize := int64(opts.FileSize)
70+
if opts.RandomSize {
71+
filesize = rand.Int63n(filesize) + 1
72+
}
73+
74+
n := rand.Intn(FilenameSize-4) + 4
75+
name := RandomFilename(n, opts.Alphabet)
76+
filepath := path.Join(root, name)
77+
f, err := os.Create(filepath)
78+
if err != nil {
79+
return err
80+
}
81+
82+
if _, err := io.CopyN(f, opts.Source, filesize); err != nil {
83+
return err
84+
}
85+
86+
if opts.Out != nil {
87+
fmt.Fprintln(opts.Out, filepath)
88+
}
89+
90+
return f.Close()
91+
}
92+
93+
func WriteRandomDir(root string, depth int, opts *Options) error {
94+
if depth > opts.FanoutDepth {
95+
return nil
96+
}
97+
98+
n := rand.Intn(FilenameSize-4) + 4
99+
name := RandomFilename(n, opts.Alphabet)
100+
root = path.Join(root, name)
101+
if err := os.MkdirAll(root, 0755); err != nil {
102+
return err
103+
}
104+
105+
if opts.Out != nil {
106+
fmt.Fprintln(opts.Out, root)
107+
}
108+
109+
return WriteRandomFiles(root, depth, opts)
110+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
random-files
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# random-files - create random fs hierarchies
2+
3+
random-files creates random fs hierarchies. Useful for testing filesystems.
4+
5+
## Install
6+
7+
```
8+
go get -u github.com/jbenet/go-random-files/random-files
9+
```
10+
11+
## Usage
12+
13+
```sh
14+
> random-files --help
15+
usage: random-files [options] <path>...
16+
Write a random filesystem hierarchy to each <path>
17+
18+
Options:
19+
-alphabet="easy": alphabet for filenames {easy, hard}
20+
-depth=2: fanout depth - how deep the hierarchy goes
21+
-dirs=5: fanout dirs - number of dirs per dir (or max)
22+
-files=10: fanout files - number of files per dir (or max
23+
-filesize=4096: filesize - how big to make each file (or max)
24+
-q=false: quiet output
25+
-random-crypto=false: use cryptographic randomness for files
26+
-random-fanout=false: randomize fanout numbers
27+
-random-size=true: randomize filesize
28+
-seed=0: random seed - 0 for current time
29+
```
30+
31+
## Examples
32+
33+
```sh
34+
> random-files --depth=2 --files=3 foo
35+
foo/h20uo3jrpihb
36+
foo/x6tef1
37+
foo/jh0c2vdci
38+
foo/fden012m368
39+
foo/fden012m368/p6n0chy4kg
40+
foo/fden012m368/h92_
41+
foo/fden012m368/kvjiya98p3
42+
foo/e_i6hwav1tb
43+
foo/e_i6hwav1tb/oj0-a
44+
foo/e_i6hwav1tb/1-pfgvim
45+
foo/e_i6hwav1tb/s_unf
46+
foo/bgvy8x-_hsm
47+
foo/bgvy8x-_hsm/98zcoz-9ng
48+
foo/bgvy8x-_hsm/j0see3qv
49+
foo/bgvy8x-_hsm/qntuf0r
50+
foo/6zjkw3ejm2awwt
51+
foo/6zjkw3ejm2awwt/iba52dh1lhnewh
52+
foo/6zjkw3ejm2awwt/n1bwcv5zpe
53+
foo/6zjkw3ejm2awwt/o8k89cc
54+
foo/efp_6
55+
foo/efp_6/qfap2
56+
foo/efp_6/v_kl_wlefsaa
57+
foo/efp_6/r7sdbph
58+
```
59+
60+
It made:
61+
62+
```
63+
> tree foo
64+
foo
65+
├── 6zjkw3ejm2awwt
66+
│   ├── iba52dh1lhnewh
67+
│   ├── n1bwcv5zpe
68+
│   └── o8k89cc
69+
├── bgvy8x-_hsm
70+
│   ├── 98zcoz-9ng
71+
│   ├── j0see3qv
72+
│   └── qntuf0r
73+
├── e_i6hwav1tb
74+
│   ├── 1-pfgvim
75+
│   ├── oj0-a
76+
│   └── s_unf
77+
├── efp_6
78+
│   ├── qfap2
79+
│   ├── r7sdbph
80+
│   └── v_kl_wlefsaa
81+
├── fden012m368
82+
│   ├── h92_
83+
│   ├── kvjiya98p3
84+
│   └── p6n0chy4kg
85+
├── h20uo3jrpihb
86+
├── jh0c2vdci
87+
└── x6tef1
88+
89+
5 directories, 18 files
90+
```

0 commit comments

Comments
 (0)