Skip to content

Commit 1b7ecc5

Browse files
lyyyunalyyyuna
authored
Merge pull request #608 from YangSen-qn/master
pfop: new type
2 parents 109f868 + 1ddc3c4 commit 1b7ecc5

File tree

10 files changed

+173
-17
lines changed

10 files changed

+173
-17
lines changed

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
# Changelog
2+
## 7.16.0（2024-09-12)
3+
* 支持闲时任务
4+
* 移除已下线区域相关域名
5+
26
## 7.15.1（2024-05-29)
37
* 处理在构造 Download URL 时 Key 前缀为 / 的情况
48

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,12 @@
1313
<dependency>
1414
<groupId>com.qiniu</groupId>
1515
<artifactId>qiniu-java-sdk</artifactId>
16-
<version>[7.15.0, 7.15.99]</version>
16+
<version>[7.16.0, 7.16.99]</version>
1717
</dependency>
1818
```
1919
或者 Gradle:
2020
```groovy
21-
implementation 'com.qiniu:qiniu-java-sdk:7.15.+'
21+
implementation 'com.qiniu:qiniu-java-sdk:7.16.+'
2222
```
2323

2424
## 运行环境

‎src/main/java/com/qiniu/common/Constants.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ public final class Constants {
1010
/**
1111
* 版本号
1212
*/
13-
public static final String VERSION = "7.15.1";
13+
public static final String VERSION = "7.16.0";
1414
/**
1515
* 块大小，不能改变
1616
*/

‎src/main/java/com/qiniu/processing/OperationManager.java‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public String pfop(String bucket, String key, String fops) throws QiniuException
8080
* @param bucket 空间名
8181
* @param key 文件名
8282
* @param fops fops指令，如果有多个指令，需要使用分号(;)进行拼接，例如 avthumb/mp4/xxx|saveas/xxx;vframe/jpg/xxx|saveas/xxx
83-
* @param params notifyURL、force、pipeline 等参数
83+
* @param params notifyURL、force、pipeline、type等参数
8484
* @return persistentId 请求返回的任务ID，可以根据该ID查询任务状态
8585
* @throws QiniuException 触发失败异常，包含错误响应等信息
8686
* <a href="http://developer.qiniu.com/dora/api/persistent-data-processing-pfop"> 相关链接 </a>
@@ -154,8 +154,34 @@ public String pfop(String bucket, String key, String fops, String pipeline, bool
154154
*/
155155
public String pfop(String bucket, String key, String fops, String pipeline, String notifyURL, boolean force)
156156
throws QiniuException {
157-
StringMap params = new StringMap().putNotEmpty("pipeline", pipeline).
158-
putNotEmpty("notifyURL", notifyURL).putWhen("force", 1, force);
157+
StringMap params = new StringMap()
158+
.putNotEmpty("pipeline", pipeline)
159+
.putNotEmpty("notifyURL", notifyURL)
160+
.putWhen("force", 1, force);
161+
return pfop(bucket, key, fops, params);
162+
}
163+
164+
/**
165+
* 发送请求对空间中的文件进行持久化处理
166+
*
167+
* @param bucket 空间名
168+
* @param key 文件名
169+
* @param fops fop指令
170+
* @param pipeline 持久化数据处理队列名称
171+
* @param notifyURL 处理结果通知地址，任务完成后自动以POST方式将处理结果提交到指定的地址
172+
* @param type 任务类型，0：非闲时任务，1：闲时任务
173+
* @param force 用于对同一个指令进行强制处理时指定，一般用于覆盖空间已有文件或者重试失败的指令
174+
* @return persistentId 请求返回的任务ID，可以根据该ID查询任务状态
175+
* @throws QiniuException 触发失败异常，包含错误响应等信息
176+
* <a href="http://developer.qiniu.com/dora/api/persistent-data-processing-pfop"> 相关链接 </a>
177+
*/
178+
public String pfop(String bucket, String key, String fops, String pipeline, String notifyURL, Integer type, boolean force)
179+
throws QiniuException {
180+
StringMap params = new StringMap()
181+
.putNotNull("type", type)
182+
.putNotEmpty("pipeline", pipeline)
183+
.putNotEmpty("notifyURL", notifyURL)
184+
.putWhen("force", 1, force);
159185
return pfop(bucket, key, fops, params);
160186
}
161187

‎src/main/java/com/qiniu/processing/OperationStatus.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,30 +9,49 @@ public class OperationStatus {
99
* 持久化处理的进程ID，即 persistentId
1010
*/
1111
public String id;
12+
1213
/**
1314
* 状态码 0 成功，1 等待处理，2 正在处理，3 处理失败，4 通知提交失败
1415
*/
1516
public int code;
17+
1618
/**
1719
* 与状态码相对应的详细描述
1820
*/
1921
public String desc;
22+
2023
/**
2124
* 处理源文件的文件名
2225
*/
2326
public String inputKey;
27+
2428
/**
2529
* 处理源文件所在的空间名
2630
*/
2731
public String inputBucket;
32+
2833
/**
2934
* 云处理操作的处理队列
3035
*/
3136
public String pipeline;
37+
3238
/**
3339
* 云处理请求的请求id，主要用于七牛技术人员的问题排查
3440
*/
3541
public String reqid;
42+
43+
/**
44+
* 是否是闲时任务
45+
* null 或 0：非闲时任务
46+
* 1：闲时任务
47+
*/
48+
public Integer type;
49+
50+
/**
51+
* 任务创建时间
52+
*/
53+
public String creationDate;
54+
3655
/**
3756
* 云处理操作列表，包含每个云处理操作的状态信息
3857
*/

‎src/main/java/com/qiniu/storage/Api.java‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,10 @@ public static class Request implements Cloneable {
395395
* @param urlPrefix 请求的 urlPrefix， scheme + host
396396
*/
397397
protected Request(String urlPrefix) {
398+
if (StringUtils.isNullOrEmpty(urlPrefix)) {
399+
return;
400+
}
401+
398402
try {
399403
URL url = new URL(urlPrefix);
400404
this.scheme = url.getProtocol();
@@ -634,6 +638,15 @@ public StringMap getHeader() throws QiniuException {
634638
return header;
635639
}
636640

641+
/**
642+
* 构造 header 信息，如果需要设置请求体，子类需要重写
643+
*
644+
* @throws QiniuException 组装 header 时的异常，一般为缺失必要参数的异常
645+
*/
646+
protected void buildHeader() throws QiniuException {
647+
648+
}
649+
637650
/**
638651
* 获取 URL
639652
*
@@ -804,6 +817,7 @@ boolean canRetry() {
804817
protected void prepareToRequest() throws QiniuException {
805818
buildPath();
806819
buildQuery();
820+
buildHeader();
807821
buildBodyInfo();
808822
}
809823

@@ -1052,14 +1066,18 @@ protected RequestBody get() {
10521066
}
10531067

10541068
final MultipartBody.Builder b = new MultipartBody.Builder();
1055-
b.addFormDataPart(name, fileName, body);
1056-
1057-
fields.forEach(new StringMap.Consumer() {
1058-
@Override
1059-
public void accept(String key, Object value) {
1060-
b.addFormDataPart(key, value.toString());
1061-
}
1062-
});
1069+
if (!StringUtils.isNullOrEmpty(name)) {
1070+
b.addFormDataPart(name, fileName, body);
1071+
}
1072+
1073+
if (fields != null) {
1074+
fields.forEach(new StringMap.Consumer() {
1075+
@Override
1076+
public void accept(String key, Object value) {
1077+
b.addFormDataPart(key, value.toString());
1078+
}
1079+
});
1080+
}
10631081

10641082
b.setType(MediaType.parse("multipart/form-data"));
10651083

‎src/main/java/com/qiniu/storage/ApiInterceptorRetryHosts.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,14 @@ Api.Response intercept(Api.Request request, Api.Handler handler) throws QiniuExc
3737
}
3838

3939
String reqHost = request.getHost();
40-
String firstHost = hostProvider.provider();
41-
if (!hostProvider.isHostValid(reqHost) && !StringUtils.isNullOrEmpty(firstHost)) {
42-
request.setHost(firstHost);
40+
if (!hostProvider.isHostValid(reqHost)) {
41+
// 支持不配置默认的 host，未配置则从 provider 中获取
42+
String firstHost = hostProvider.provider();
43+
if (!StringUtils.isNullOrEmpty(firstHost)) {
44+
request.setHost(firstHost);
45+
} else {
46+
throw QiniuException.unrecoverable("no host provide");
47+
}
4348
}
4449

4550
if (retryMax == 0) {

‎src/main/java/com/qiniu/util/Auth.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ public final class Auth {
4848
"fsizeMin",
4949
"trafficLimit",
5050

51+
"persistentType",
5152
"persistentOps",
5253
"persistentNotifyUrl",
5354
"persistentPipeline",

‎src/test/java/test/com/qiniu/processing/PfopTest.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import com.qiniu.processing.OperationStatus;
88
import com.qiniu.storage.Configuration;
99
import com.qiniu.storage.Region;
10+
import com.qiniu.util.Auth;
1011
import com.qiniu.util.StringUtils;
1112
import com.qiniu.util.UrlSafeBase64;
1213
import org.junit.jupiter.api.Tag;
@@ -138,4 +139,34 @@ private void testPfopIsSuccess(String jobid) {
138139
assertEquals(0, status.code);
139140
}
140141

142+
@Test
143+
@Tag("IntegrationTest")
144+
void testPfopWithType() {
145+
try {
146+
Auth auth = TestConfig.testAuth;
147+
Map<String, Region> bucketKeyMap = new HashMap<String, Region>();
148+
TestConfig.TestFile[] files = TestConfig.getTestFileArray();
149+
for (TestConfig.TestFile testFile : files) {
150+
bucketKeyMap.put(testFile.getBucketName(), testFile.getRegion());
151+
}
152+
for (Map.Entry<String, Region> entry : bucketKeyMap.entrySet()) {
153+
String bucket = entry.getKey();
154+
Region region = entry.getValue();
155+
156+
Configuration cfg = new Configuration(region);
157+
OperationManager operationManager = new OperationManager(auth, cfg);
158+
String jobID = operationManager.pfop(bucket, TestConfig.testMp4FileKey, "avinfo", "", "", 1, true);
159+
160+
OperationStatus status = operationManager.prefop(bucket, jobID);
161+
assertNotNull(status, "1. prefop type error");
162+
assertNotNull(status.creationDate, "1. prefop type error");
163+
assertTrue(status.code == 0 || status.code == 1 || status.code == 3, "2. prefop type error");
164+
assertEquals(1, (int) status.type, "3. prefop type error");
165+
}
166+
167+
} catch (QiniuException ex) {
168+
ex.printStackTrace();
169+
assertTrue(ResCode.find(ex.code(), ResCode.getPossibleResCode(612)));
170+
}
171+
}
141172
}

‎src/test/java/test/com/qiniu/storage/FormUploadTest.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
import com.qiniu.common.QiniuException;
44
import com.qiniu.http.Response;
5+
import com.qiniu.processing.OperationManager;
6+
import com.qiniu.processing.OperationStatus;
57
import com.qiniu.storage.Configuration;
68
import com.qiniu.storage.UpCompletionHandler;
79
import com.qiniu.storage.UploadManager;
@@ -27,6 +29,56 @@ public class FormUploadTest {
2729

2830
UploadManager uploadManager = new UploadManager(new Configuration());
2931

32+
@Test
33+
@Tag("IntegrationTest")
34+
void testUploadWithFop() {
35+
TestConfig.TestFile file = TestConfig.getTestFileArray()[0];
36+
final String expectKey = "test-fop";
37+
final String bucket = file.getBucketName();
38+
39+
String persistentOpfs = String.format("%s:vframe_test_target.jpg", bucket);
40+
StringMap policy = new StringMap();
41+
policy.put("persistentOps", persistentOpfs);
42+
policy.put("persistentType", 1);
43+
44+
Configuration config = new Configuration();
45+
config.useHttpsDomains = true;
46+
47+
Response r = null;
48+
try {
49+
String token = TestConfig.testAuth.uploadToken(bucket, expectKey, 3600, policy);
50+
UploadManager uploadManager = new UploadManager(config);
51+
StringMap params = new StringMap().put("x:foo", "foo_val");
52+
r = uploadManager.put("hello".getBytes(), expectKey, token, params, null, false);
53+
} catch (QiniuException e) {
54+
fail(e.toString());
55+
}
56+
assertEquals(200, r.statusCode);
57+
58+
StringMap map = null;
59+
try {
60+
map = r.jsonToMap();
61+
} catch (QiniuException e) {
62+
fail(e.toString());
63+
}
64+
65+
assertNotNull(map, "1. testUploadWithFop error");
66+
67+
String persistentId = (String) map.get("persistentId");
68+
assertNotNull(persistentId, "2. testUploadWithFop error");
69+
70+
try {
71+
OperationManager operationManager = new OperationManager(TestConfig.testAuth, config);
72+
OperationStatus status = operationManager.prefop(bucket, persistentId);
73+
assertNotNull(status, "3. prefop type error");
74+
assertNotNull(status.creationDate, "4. prefop type error");
75+
assertTrue(status.code == 0 || status.code == 1 || status.code == 3, "5. prefop type error");
76+
assertEquals(1, (int) status.type, "6. prefop type error");
77+
} catch (QiniuException e) {
78+
fail(e.toString());
79+
}
80+
}
81+
3082
/**
3183
* hello上传测试
3284
*/

0 commit comments

Comments
 (0)