Skip to content

Commit 5bb70c3

Browse files
authored
upload with more Put policy (#413)
1 parent 0e4cf4e commit 5bb70c3

File tree

15 files changed

+326
-57
lines changed

15 files changed

+326
-57
lines changed

cmd/upload.go

Lines changed: 78 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package cmd
22

33
import (
4-
"github.com/qiniu/go-sdk/v7/storage"
54
"github.com/spf13/cobra"
65

76
"github.com/qiniu/qshell/v2/docs"
@@ -48,9 +47,7 @@ var upload2CmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
4847
LogRotate = 7
4948
)
5049
info := operations.BatchUpload2Info{
51-
UploadConfig: operations.UploadConfig{
52-
Policy: &storage.PutPolicy{},
53-
},
50+
UploadConfig: operations.UploadConfig{},
5451
}
5552
cmd := &cobra.Command{
5653
Use: "qupload2",
@@ -102,19 +99,34 @@ var upload2CmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
10299
cmd.Flags().IntVarP(&info.FileType, "storage", "", 0, "set storage type of file, same to --file-type")
103100
_ = cmd.Flags().MarkDeprecated("storage", "use --file-type instead") // 废弃 storage
104101

105-
cmd.Flags().StringVarP(&info.Policy.CallbackURL, "callback-urls", "l", "", "upload callback urls, separated by comma")
106-
cmd.Flags().StringVarP(&info.Policy.CallbackHost, "callback-host", "T", "", "upload callback host")
107102
//cmd.Flags().StringVar(&cfg.CmdCfg.Up.BindUpIp, "bind-up-ip", "", "upload host ip to bind")
108103
//cmd.Flags().StringVar(&cfg.CmdCfg.Up.BindRsIp, "bind-rs-ip", "", "rs host ip to bind")
109104
//cmd.Flags().StringVar(&cfg.CmdCfg.Up.BindNicIp, "bind-nic-ip", "", "local network interface card to bind")
110105

106+
cmd.Flags().StringVarP(&info.EndUser, "end-user", "", "", "Owner identification")
107+
cmd.Flags().StringVarP(&info.CallbackURL, "callback-urls", "l", "", "upload callback urls, separated by comma")
108+
cmd.Flags().StringVarP(&info.CallbackHost, "callback-host", "T", "", "upload callback host")
109+
cmd.Flags().StringVarP(&info.CallbackBody, "callback-body", "", "", "upload callback body")
110+
cmd.Flags().StringVarP(&info.CallbackBodyType, "callback-body-type", "", "", "upload callback body type")
111+
cmd.Flags().StringVarP(&info.PersistentOps, "persistent-ops", "", "", "List of pre-transfer persistence processing instructions that are triggered after successful resource upload. This parameter is not supported when fileType=2 or 3 (upload archive storage or deep archive storage files). Supports magic variables and custom variables. Each directive is an API specification string, and multiple directives are separated by ;.")
112+
cmd.Flags().StringVarP(&info.PersistentNotifyURL, "persistent-notify-url", "", "", "URL to receive notification of persistence processing results. It must be a valid URL that can make POST requests normally on the public Internet and respond successfully. The content obtained by this URL is consistent with the processing result of the persistence processing status query. To send a POST request whose body format is application/json, you need to read the body of the request in the form of a read stream to obtain it.")
113+
cmd.Flags().StringVarP(&info.PersistentPipeline, "persistent-pipeline", "", "", "Transcoding queue name. After the resource is successfully uploaded, an independent queue is designated for transcoding when transcoding is triggered. If it is empty, it means that the public queue is used, and the processing speed is slower. It is recommended to use a dedicated queue.")
114+
cmd.Flags().IntVarP(&info.DetectMime, "detect-mime", "", 0, `Turn on the MimeType detection function and perform detection according to the following rules; if the correct value cannot be detected, application/octet-stream will be used by default.
115+
If set to a value of 1, the file MimeType information passed by the uploader will be ignored, and the MimeType value will be detected in the following order:
116+
1. Detection content;
117+
2. Check the file extension;
118+
3. Check the Key extension.
119+
The default value is set to 0.If the uploader specifies MimeType (except application/octet-stream), this value will be used directly. Otherwise, the MimeType value will be detected in the following order:
120+
1. Check the file extension;
121+
2. Check the Key extension;
122+
3. Detect content.
123+
Set to a value of -1 and use this value regardless of what value is specified on the uploader.`)
124+
cmd.Flags().Uint64VarP(&info.TrafficLimit, "traffic-limit", "", 0, "Upload request single link speed limit to control client bandwidth usage. The speed limit value range is 819200 ~ 838860800, and the unit is bit/s.")
111125
return cmd
112126
}
113127

114128
var syncCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
115-
info := operations.SyncInfo{
116-
Policy: &storage.PutPolicy{},
117-
}
129+
info := operations.SyncInfo{}
118130
cmd := &cobra.Command{
119131
Use: "sync <SrcResUrl> <Buckets> [-k <Key>]",
120132
Short: "Sync big file to qiniu bucket",
@@ -141,13 +153,30 @@ var syncCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
141153

142154
cmd.Flags().BoolVarP(&info.Overwrite, "overwrite", "", false, "overwrite the file of same key in bucket")
143155

156+
cmd.Flags().StringVarP(&info.Policy.EndUser, "end-user", "", "", "Owner identification")
157+
cmd.Flags().StringVarP(&info.Policy.CallbackURL, "callback-urls", "l", "", "upload callback urls, separated by comma")
158+
cmd.Flags().StringVarP(&info.Policy.CallbackHost, "callback-host", "T", "", "upload callback host")
159+
cmd.Flags().StringVarP(&info.Policy.CallbackBody, "callback-body", "", "", "upload callback body")
160+
cmd.Flags().StringVarP(&info.Policy.CallbackBodyType, "callback-body-type", "", "", "upload callback body type")
161+
cmd.Flags().StringVarP(&info.Policy.PersistentOps, "persistent-ops", "", "", "List of pre-transfer persistence processing instructions that are triggered after successful resource upload. This parameter is not supported when fileType=2 or 3 (upload archive storage or deep archive storage files). Supports magic variables and custom variables. Each directive is an API specification string, and multiple directives are separated by ;.")
162+
cmd.Flags().StringVarP(&info.Policy.PersistentNotifyURL, "persistent-notify-url", "", "", "URL to receive notification of persistence processing results. It must be a valid URL that can make POST requests normally on the public Internet and respond successfully. The content obtained by this URL is consistent with the processing result of the persistence processing status query. To send a POST request whose body format is application/json, you need to read the body of the request in the form of a read stream to obtain it.")
163+
cmd.Flags().StringVarP(&info.Policy.PersistentPipeline, "persistent-pipeline", "", "", "Transcoding queue name. After the resource is successfully uploaded, an independent queue is designated for transcoding when transcoding is triggered. If it is empty, it means that the public queue is used, and the processing speed is slower. It is recommended to use a dedicated queue.")
164+
cmd.Flags().IntVarP(&info.Policy.DetectMime, "detect-mime", "", 0, `Turn on the MimeType detection function and perform detection according to the following rules; if the correct value cannot be detected, application/octet-stream will be used by default.
165+
If set to a value of 1, the file MimeType information passed by the uploader will be ignored, and the MimeType value will be detected in the following order:
166+
1. Detection content;
167+
2. Check the file extension;
168+
3. Check the Key extension.
169+
The default value is set to 0.If the uploader specifies MimeType (except application/octet-stream), this value will be used directly. Otherwise, the MimeType value will be detected in the following order:
170+
1. Check the file extension;
171+
2. Check the Key extension;
172+
3. Detect content.
173+
Set to a value of -1 and use this value regardless of what value is specified on the uploader.`)
174+
cmd.Flags().Uint64VarP(&info.Policy.TrafficLimit, "traffic-limit", "", 0, "Upload request single link speed limit to control client bandwidth usage. The speed limit value range is 819200 ~ 838860800, and the unit is bit/s.")
144175
return cmd
145176
}
146177

147178
var formUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
148-
info := operations.UploadInfo{
149-
Policy: &storage.PutPolicy{},
150-
}
179+
info := operations.UploadInfo{}
151180
cmd := &cobra.Command{
152181
Use: "fput <Bucket> <Key> <LocalFile>",
153182
Short: "Form upload a local file",
@@ -174,15 +203,31 @@ var formUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
174203
_ = cmd.Flags().MarkDeprecated("storage", "use --file-type instead") // 废弃 storage
175204

176205
cmd.Flags().StringVarP(&info.UpHost, "up-host", "u", "", "uphost")
206+
207+
cmd.Flags().StringVarP(&info.Policy.EndUser, "end-user", "", "", "Owner identification")
177208
cmd.Flags().StringVarP(&info.Policy.CallbackURL, "callback-urls", "l", "", "upload callback urls, separated by comma")
178209
cmd.Flags().StringVarP(&info.Policy.CallbackHost, "callback-host", "T", "", "upload callback host")
210+
cmd.Flags().StringVarP(&info.Policy.CallbackBody, "callback-body", "", "", "upload callback body")
211+
cmd.Flags().StringVarP(&info.Policy.CallbackBodyType, "callback-body-type", "", "", "upload callback body type")
212+
cmd.Flags().StringVarP(&info.Policy.PersistentOps, "persistent-ops", "", "", "List of pre-transfer persistence processing instructions that are triggered after successful resource upload. This parameter is not supported when fileType=2 or 3 (upload archive storage or deep archive storage files). Supports magic variables and custom variables. Each directive is an API specification string, and multiple directives are separated by ;.")
213+
cmd.Flags().StringVarP(&info.Policy.PersistentNotifyURL, "persistent-notify-url", "", "", "URL to receive notification of persistence processing results. It must be a valid URL that can make POST requests normally on the public Internet and respond successfully. The content obtained by this URL is consistent with the processing result of the persistence processing status query. To send a POST request whose body format is application/json, you need to read the body of the request in the form of a read stream to obtain it.")
214+
cmd.Flags().StringVarP(&info.Policy.PersistentPipeline, "persistent-pipeline", "", "", "Transcoding queue name. After the resource is successfully uploaded, an independent queue is designated for transcoding when transcoding is triggered. If it is empty, it means that the public queue is used, and the processing speed is slower. It is recommended to use a dedicated queue.")
215+
cmd.Flags().IntVarP(&info.Policy.DetectMime, "detect-mime", "", 0, `Turn on the MimeType detection function and perform detection according to the following rules; if the correct value cannot be detected, application/octet-stream will be used by default.
216+
If set to a value of 1, the file MimeType information passed by the uploader will be ignored, and the MimeType value will be detected in the following order:
217+
1. Detection content;
218+
2. Check the file extension;
219+
3. Check the Key extension.
220+
The default value is set to 0.If the uploader specifies MimeType (except application/octet-stream), this value will be used directly. Otherwise, the MimeType value will be detected in the following order:
221+
1. Check the file extension;
222+
2. Check the Key extension;
223+
3. Detect content.
224+
Set to a value of -1 and use this value regardless of what value is specified on the uploader.`)
225+
cmd.Flags().Uint64VarP(&info.Policy.TrafficLimit, "traffic-limit", "", 0, "Upload request single link speed limit to control client bandwidth usage. The speed limit value range is 819200 ~ 838860800, and the unit is bit/s.")
179226
return cmd
180227
}
181228

182229
var resumeUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
183-
info := operations.UploadInfo{
184-
Policy: &storage.PutPolicy{},
185-
}
230+
info := operations.UploadInfo{}
186231
cmd := &cobra.Command{
187232
Use: "rput <Bucket> <Key> <LocalFile>",
188233
Short: "Resumable upload a local file",
@@ -217,8 +262,26 @@ var resumeUploadCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
217262

218263
cmd.Flags().IntVarP(&info.ResumeWorkerCount, "worker", "c", 3, "worker count")
219264
cmd.Flags().StringVarP(&info.UpHost, "up-host", "u", "", "uphost")
265+
266+
cmd.Flags().StringVarP(&info.Policy.EndUser, "end-user", "", "", "Owner identification")
220267
cmd.Flags().StringVarP(&info.Policy.CallbackURL, "callback-urls", "l", "", "upload callback urls, separated by comma")
221268
cmd.Flags().StringVarP(&info.Policy.CallbackHost, "callback-host", "T", "", "upload callback host")
269+
cmd.Flags().StringVarP(&info.Policy.CallbackBody, "callback-body", "", "", "upload callback body")
270+
cmd.Flags().StringVarP(&info.Policy.CallbackBodyType, "callback-body-type", "", "", "upload callback body type")
271+
cmd.Flags().StringVarP(&info.Policy.PersistentOps, "persistent-ops", "", "", "List of pre-transfer persistence processing instructions that are triggered after successful resource upload. This parameter is not supported when fileType=2 or 3 (upload archive storage or deep archive storage files). Supports magic variables and custom variables. Each directive is an API specification string, and multiple directives are separated by ;.")
272+
cmd.Flags().StringVarP(&info.Policy.PersistentNotifyURL, "persistent-notify-url", "", "", "URL to receive notification of persistence processing results. It must be a valid URL that can make POST requests normally on the public Internet and respond successfully. The content obtained by this URL is consistent with the processing result of the persistence processing status query. To send a POST request whose body format is application/json, you need to read the body of the request in the form of a read stream to obtain it.")
273+
cmd.Flags().StringVarP(&info.Policy.PersistentPipeline, "persistent-pipeline", "", "", "Transcoding queue name. After the resource is successfully uploaded, an independent queue is designated for transcoding when transcoding is triggered. If it is empty, it means that the public queue is used, and the processing speed is slower. It is recommended to use a dedicated queue.")
274+
cmd.Flags().IntVarP(&info.Policy.DetectMime, "detect-mime", "", 0, `Turn on the MimeType detection function and perform detection according to the following rules; if the correct value cannot be detected, application/octet-stream will be used by default.
275+
If set to a value of 1, the file MimeType information passed by the uploader will be ignored, and the MimeType value will be detected in the following order:
276+
1. Detection content;
277+
2. Check the file extension;
278+
3. Check the Key extension.
279+
The default value is set to 0. If the uploader specifies MimeType (except application/octet-stream), this value will be used directly. Otherwise, the MimeType value will be detected in the following order:
280+
1. Check the file extension;
281+
2. Check the Key extension;
282+
3. Detect content.
283+
Set to a value of -1 and use this value regardless of what value is specified on the uploader.`)
284+
cmd.Flags().Uint64VarP(&info.Policy.TrafficLimit, "traffic-limit", "", 0, "Upload request single link speed limit to control client bandwidth usage. The speed limit value range is 819200 ~ 838860800, and the unit is bit/s.")
222285
return cmd
223286
}
224287

cmd_test/upload_form_test.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,10 @@
33
package cmd
44

55
import (
6-
"github.com/qiniu/qshell/v2/cmd_test/test"
76
"strings"
87
"testing"
8+
9+
"github.com/qiniu/qshell/v2/cmd_test/test"
910
)
1011

1112
func TestFormUpload(t *testing.T) {
@@ -19,7 +20,8 @@ func TestFormUpload(t *testing.T) {
1920
result, errs := test.RunCmdWithError("fput", test.Bucket, "qshell_fput_1M", path,
2021
"--mimetype", "image/jpg",
2122
"--storage", "0",
22-
"--file-type", "1")
23+
"--file-type", "1",
24+
"--end-user", "10")
2325
if len(errs) > 0 {
2426
t.Fail()
2527
}

cmd_test/upload_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@ package cmd
44

55
import (
66
"encoding/json"
7-
"github.com/qiniu/qshell/v2/cmd_test/test"
8-
"github.com/qiniu/qshell/v2/iqshell/storage/object/upload/operations"
97
"path/filepath"
108
"strings"
119
"testing"
10+
11+
"github.com/qiniu/qshell/v2/cmd_test/test"
12+
"github.com/qiniu/qshell/v2/iqshell/storage/object/upload/operations"
1213
)
1314

1415
func TestQUpload(t *testing.T) {
@@ -71,7 +72,6 @@ func TestQUpload(t *testing.T) {
7172
DisableForm: false,
7273
WorkerCount: 4,
7374
RecordRoot: "",
74-
Policy: nil,
7575
},
7676
LogFile: logPath,
7777
RecordRoot: recordPath,

docs/fput.md

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,26 @@ $ qshell fput --doc
3535
- -u/--up-host: 指定上传域名。 【可选】
3636
- -l/--callback-urls:上传回调地址, 可以指定多个地址,以逗号分隔。 【可选】
3737
- -T/--callback-host:上传回调的 HOST, 必须和 CallbackUrls 一起指定。 【可选】
38+
- --callback-body:上传成功后,七牛云向业务服务器发送 Content-Type: application/x-www-form-urlencoded 的 POST 请求。业务服务器可以通过直接读取请求的 query 来获得该字段,支持魔法变量和自定义变量。callbackBody 要求是合法的 url query string。例如key=$(key)&hash=$(etag)&w=$(imageInfo.width)&h=$(imageInfo.height)。如果callbackBodyType指定为application/json,则callbackBody应为json格式,例如:{“key”:"$(key)",“hash”:"$(etag)",“w”:"$(imageInfo.width)",“h”:"$(imageInfo.height)"}。【可选】
39+
- --callback-body-type:上传成功后,七牛云向业务服务器发送回调通知 callbackBody 的 Content-Type。默认为 application/x-www-form-urlencoded,也可设置为 application/json。【可选】
40+
- --end-user:上传成功后,七牛云向业务服务器发送回调通知 callbackBody 的 Content-Type。默认为 application/x-www-form-urlencoded,也可设置为 application/json。【可选】
41+
- --persistent-ops:资源上传成功后触发执行的预转持久化处理指令列表。fileType=2或3(上传归档存储或深度归档存储文件)时,不支持使用该参数。支持魔法变量和自定义变量。每个指令是一个 API 规格字符串,多个指令用;分隔。【可选】
42+
- --persistent-notify-url:接收持久化处理结果通知的 URL。必须是公网上可以正常进行 POST 请求并能成功响应的有效 URL。该 URL 获取的内容和持久化处理状态查询的处理结果一致。发送 body 格式是 Content-Type 为 application/json 的 POST 请求,需要按照读取流的形式读取请求的 body 才能获取。【可选】
43+
- --persistent-pipeline:转码队列名。资源上传成功后,触发转码时指定独立的队列进行转码。为空则表示使用公用队列,处理速度比较慢。建议使用专用队列。【可选】
44+
- --detect-mime:开启 MimeType 侦测功能,并按照下述规则进行侦测;如不能侦测出正确的值,会默认使用 application/octet-stream 。【可选】
45+
```
46+
1. 设为 1 值,则忽略上传端传递的文件 MimeType 信息,并按如下顺序侦测 MimeType 值:
47+
1) 侦测内容;
48+
2) 检查文件扩展名;
49+
3) 检查 Key 扩展名。
50+
2. 默认设为 0 值,如上传端指定了 MimeType(application/octet-stream 除外)则直接使用该值,否则按如下顺序侦测 MimeType 值:
51+
1) 检查文件扩展名;
52+
2) 检查 Key 扩展名;
53+
3) 侦测内容。
54+
3. 设为 -1 值,无论上传端指定了何值直接使用该值。
55+
```
56+
- --traffic-limit:上传请求单链接速度限制,控制客户端带宽占用。限速值取值范围为 819200 ~ 838860800,单位为 bit/s。【可选】
57+
3858

3959
# 示例
4060
1 上传本地文件 `/Users/jemy/Documents/qiniu.jpg` 到空间 `if-pbl` 里面。

0 commit comments

Comments
 (0)