Skip to content

Commit 87e1d1d

Browse files
klauspostklauspost
authored
Add S3 Express Append benchmark (#405)
Each thread uploads one object and appends to it.
1 parent 6d70273 commit 87e1d1d

File tree

4 files changed

+283
-0
lines changed

4 files changed

+283
-0
lines changed

‎README.md‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -546,6 +546,43 @@ Throughput, split into 58 x 1s:
546546
Cleanup Done
547547
```
548548

549+
## APPEND (S3 Express)
550+
551+
Benchmarks S3 Express One Zone [Append Object](https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-buckets-objects-append.html) operations.
552+
553+
WARP will upload `--obj.size` objects for each `--concurrent` and append up to 10,000 parts to these.
554+
Each append operation will be one part and the size of each part will be `--part.size` - a new object will be created when the part limit is reached.
555+
556+
If no `--checksum` is specified, the CRC64NVME checksum will be used. The checksum type must support full object checksums (CRC32, CRC32C, CRC64NVME).
557+
558+
Example:
559+
560+
```
561+
λ warp append -duration=1m -obj.size=1MB
562+
╭─────────────────────────────────╮
563+
│ WARP S3 Benchmark Tool by MinIO │
564+
╰─────────────────────────────────╯
565+
566+
Benchmarking: Press 'q' to abort benchmark and print partial results...
567+
568+
λ ████████████████████████████████████████████████████████████████████████░ 99%
569+
570+
Reqs: 4997, Errs:0, Objs:4997, Bytes: 4765.5MiB
571+
- APPEND Average: 84 Obj/s, 80.4MiB/s; Current 88 Obj/s, 84.4MiB/s, 280.7 ms/req
572+
573+
574+
Report: APPEND. Concurrency: 20. Ran: 58s
575+
* Average: 80.15 MiB/s, 84.04 obj/s
576+
* Reqs: Avg: 234.6ms, 50%: 203.9ms, 90%: 354.1ms, 99%: 711.3ms, Fastest: 58.3ms, Slowest: 1213.9ms, StdDev: 109.5ms
577+
578+
Throughput, split into 58 x 1s:
579+
* Fastest: 123.8MiB/s, 129.80 obj/s
580+
* 50% Median: 80.1MiB/s, 83.97 obj/s
581+
* Slowest: 23.6MiB/s, 24.74 obj/s
582+
```
583+
584+
The "obj/s" indicates the number of append operations per second.
585+
549586
## ZIP
550587

551588
The `zip` command benchmarks the MinIO [s3zip](https://blog.min.io/small-file-archives/) extension

‎cli/append.go‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*
2+
* Warp (C) 2019-2025 MinIO, Inc.
3+
*
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU Affero General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU Affero General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU Affero General Public License
15+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
18+
package cli
19+
20+
import (
21+
"fmt"
22+
23+
"github.com/minio/cli"
24+
"github.com/minio/minio-go/v7"
25+
"github.com/minio/pkg/v3/console"
26+
"github.com/minio/warp/pkg/bench"
27+
)
28+
29+
var appendFlags = []cli.Flag{
30+
cli.StringFlag{
31+
Name: "obj.size",
32+
Value: "10MiB",
33+
Usage: "Size of each append operation. Can be a number or 10KiB/MiB/GiB. All sizes are base 2 binary.",
34+
},
35+
}
36+
37+
var AppendCombinedFlags = combineFlags(globalFlags, ioFlags, appendFlags, genFlags, benchFlags, analyzeFlags)
38+
39+
// Put command.
40+
var appendCmd = cli.Command{
41+
Name: "append",
42+
Usage: "benchmark appen objects (s3 express)",
43+
Action: mainAppend,
44+
Before: setGlobalsFromContext,
45+
Flags: AppendCombinedFlags,
46+
CustomHelpTemplate: `NAME:
47+
{{.HelpName}} - {{.Usage}}
48+
49+
USAGE:
50+
{{.HelpName}} [FLAGS]
51+
-> see https://github.com/minio/warp#put
52+
53+
FLAGS:
54+
{{range .VisibleFlags}}{{.}}
55+
{{end}}`,
56+
}
57+
58+
// mainPut is the entry point for cp command.
59+
func mainAppend(ctx *cli.Context) error {
60+
checkAppendSyntax(ctx)
61+
useTrailingHeaders.Store(true)
62+
b := bench.Append{
63+
Common: getCommon(ctx, newGenSource(ctx, "obj.size")),
64+
}
65+
if b.Versioned {
66+
return fmt.Errorf("append versioned objects is not supported")
67+
}
68+
switch {
69+
case !b.PutOpts.Checksum.IsSet():
70+
// Set checksum to CRC64NVME if not set
71+
b.PutOpts.Checksum = minio.ChecksumCRC64NVME
72+
case !b.PutOpts.Checksum.CanMergeCRC():
73+
return fmt.Errorf("append benchmark requires a checksum that can merge CRC")
74+
default:
75+
// Ensure the full object checksum is set
76+
b.PutOpts.Checksum |= minio.ChecksumFullObject
77+
}
78+
79+
return runBench(ctx, &b)
80+
}
81+
82+
func checkAppendSyntax(ctx *cli.Context) {
83+
if ctx.NArg() > 0 {
84+
console.Fatal("Command takes no arguments")
85+
}
86+
87+
checkAnalyze(ctx)
88+
checkBenchmark(ctx)
89+
}

‎cli/cli.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ func init() {
100100
zipCmd,
101101
snowballCmd,
102102
fanoutCmd,
103+
appendCmd,
103104
}
104105
b := []cli.Command{
105106
analyzeCmd,

‎pkg/bench/append.go‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
/*
2+
* Warp (C) 2019-2025 MinIO, Inc.
3+
*
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU Affero General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU Affero General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU Affero General Public License
15+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
18+
package bench
19+
20+
import (
21+
"context"
22+
"fmt"
23+
"sync"
24+
"time"
25+
26+
"github.com/minio/minio-go/v7"
27+
)
28+
29+
// Append benchmarks upload speed via appends.
30+
type Append struct {
31+
Common
32+
prefixes map[string]struct{}
33+
}
34+
35+
// Prepare will create an empty bucket or delete any content already there.
36+
func (u *Append) Prepare(ctx context.Context) error {
37+
return u.createEmptyBucket(ctx)
38+
}
39+
40+
// Start will execute the main benchmark.
41+
// Operations should begin executing when the start channel is closed.
42+
func (u *Append) Start(ctx context.Context, wait chan struct{}) error {
43+
var wg sync.WaitGroup
44+
wg.Add(u.Concurrency)
45+
c := u.Collector
46+
if u.AutoTermDur > 0 {
47+
ctx = c.AutoTerm(ctx, "APPEND", u.AutoTermScale, autoTermCheck, autoTermSamples, u.AutoTermDur)
48+
}
49+
u.prefixes = make(map[string]struct{}, u.Concurrency)
50+
51+
// Non-terminating context.
52+
nonTerm := context.Background()
53+
54+
for i := 0; i < u.Concurrency; i++ {
55+
src := u.Source()
56+
u.prefixes[src.Prefix()] = struct{}{}
57+
go func(i int) {
58+
part := 1
59+
tmp := src.Object()
60+
masterObj := *tmp
61+
62+
rcv := c.Receiver()
63+
defer wg.Done()
64+
65+
// Copy usermetadata and usertags per concurrent thread.
66+
opts := u.PutOpts
67+
opts.UserMetadata = make(map[string]string, len(u.PutOpts.UserMetadata))
68+
opts.UserTags = make(map[string]string, len(u.PutOpts.UserTags))
69+
// Only create 1 part on initial upload.
70+
opts.DisableMultipart = true
71+
for k, v := range u.PutOpts.UserMetadata {
72+
opts.UserMetadata[k] = v
73+
}
74+
for k, v := range u.PutOpts.UserTags {
75+
opts.UserTags[k] = v
76+
}
77+
aOpts := minio.AppendObjectOptions{
78+
Progress: nil,
79+
ChunkSize: 0,
80+
DisableContentSha256: opts.DisableContentSha256,
81+
}
82+
83+
done := ctx.Done()
84+
85+
<-wait
86+
for {
87+
if part >= 10000 {
88+
tmp := src.Object()
89+
masterObj = *tmp
90+
part = 1
91+
}
92+
93+
select {
94+
case <-done:
95+
return
96+
default:
97+
}
98+
99+
if u.rpsLimit(ctx) != nil {
100+
return
101+
}
102+
obj := src.Object()
103+
obj.Name = masterObj.Name
104+
obj.Prefix = masterObj.Prefix
105+
obj.ContentType = masterObj.ContentType
106+
107+
opts.ContentType = obj.ContentType
108+
client, cldone := u.Client()
109+
op := Operation{
110+
OpType: "APPEND",
111+
Thread: uint16(i),
112+
Size: obj.Size,
113+
ObjPerOp: 1,
114+
File: obj.Name,
115+
Endpoint: client.EndpointURL().String(),
116+
}
117+
118+
op.Start = time.Now()
119+
var err error
120+
var res minio.UploadInfo
121+
if part == 1 {
122+
res, err = client.PutObject(nonTerm, u.Bucket, obj.Name, obj.Reader, obj.Size, opts)
123+
} else {
124+
res, err = client.AppendObject(nonTerm, u.Bucket, obj.Name, obj.Reader, obj.Size, aOpts)
125+
}
126+
op.End = time.Now()
127+
if err != nil {
128+
u.Error("upload error: ", err)
129+
op.Err = err.Error()
130+
}
131+
132+
if res.Size != int64(part)*obj.Size && op.Err == "" {
133+
err := fmt.Sprint("part ", part, " short upload. want:", int64(part)*obj.Size, ", got:", res.Size)
134+
if op.Err == "" {
135+
op.Err = err
136+
}
137+
u.Error(err)
138+
}
139+
part++
140+
cldone()
141+
rcv <- op
142+
}
143+
}(i)
144+
}
145+
wg.Wait()
146+
return nil
147+
}
148+
149+
// Cleanup deletes everything uploaded to the bucket.
150+
func (u *Append) Cleanup(ctx context.Context) {
151+
pf := make([]string, 0, len(u.prefixes))
152+
for p := range u.prefixes {
153+
pf = append(pf, p)
154+
}
155+
u.deleteAllInBucket(ctx, pf...)
156+
}

0 commit comments

Comments
 (0)