Skip to content

Commit 0846876

Browse files
gawen957gawen957
authored
Merge pull request #307 from NigelWu95/dev
fix procedure log process.
2 parents cedba91 + 76ca00f commit 0846876

File tree

8 files changed

+80
-50
lines changed

8 files changed

+80
-50
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version=8.4.7
1+
version=8.4.8
22

33
package_no_test:
44
mvn clean package -Dmaven.test.skip=true

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>com.qiniu</groupId>
88
<artifactId>qsuits</artifactId>
9-
<version>8.4.7</version>
9+
<version>8.4.8</version>
1010
<name>qsuits</name>
1111
<description>qiniu-suits is a efficient tools for qiniu api implemented by java8.</description>
1212
<url>https://github.com/NigelWu95/qiniu-suits-java</url>

‎src/main/java/com/qiniu/datasource/CloudStorageContainer.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -268,8 +268,7 @@ void listing(IStorageLister<E> lister) {
268268
processorMap.put(orderStr, lineProcessor);
269269
}
270270
export(lister, saver, lineProcessor);
271-
procedureLogger.info("{}-|-", lister.getPrefix());
272-
progressMap.remove(lister.getPrefix()); // 只有 export 成功情况下才移除 record
271+
removeRecordedPrefix(lister.getPrefix()); // 只有 export 成功情况下才移除 record
273272
} catch (QiniuException e) {
274273
try { FileUtils.createIfNotExists(errorLogFile); } catch (IOException ignored) {}
275274
errorLogger.error("{}: {}, {}", lister.getPrefix(), progressMap.get(lister.getPrefix()), e.error(), e);
@@ -383,7 +382,7 @@ private List<IStorageLister<E>> filteredListerByPrefixes(Stream<String> prefixes
383382
if (generated == null) return false;
384383
else if (generated.currents().size() > 0 || generated.hasNext()) return true;
385384
else {
386-
progressMap.remove(generated.getPrefix());
385+
removeRecordedPrefix(generated.getPrefix());
387386
generated.close();
388387
return false;
389388
}
@@ -412,7 +411,7 @@ private void processNodeLister(IStorageLister<E> lister) {
412411
if (lister.currents().size() > 0 || lister.hasNext()) {
413412
executorPool.execute(() -> listing(lister));
414413
} else {
415-
progressMap.remove(lister.getPrefix());
414+
removeRecordedPrefix(lister.getPrefix());
416415
lister.close();
417416
}
418417
}
@@ -483,12 +482,13 @@ private List<String> checkListerInPool(List<IStorageLister<E>> listerList, int c
483482
// lister 的 prefix 为 final 对象，不能因为 truncate 的操作之后被修改
484483
prefix = lister.getPrefix();
485484
nextMarker = lister.truncate();
486-
// 防止 truncate 过程中原来的线程中丢失了 prefixAndEndedMap 的操作，这里再判断一次
487485
endMap = prefixAndEndedMap.get(prefix);
488486
prefixMap = new HashMap<>();
489487
if (endMap == null) {
488+
// 做标记，endMap 为空说明该 prefix 本就不是尾结点，重新组合列举时不能因为排序最后而加入到 prefixEndedMap 中去
490489
prefixMap.put("remove", "remove");
491490
} else {
491+
// 防止 truncate 过程中原来的线程中丢失了 prefixAndEndedMap 的操作，这里再判断一次 endKey 进行更新
492492
start = lister.currentEndKey();
493493
if (start != null) endMap.put("start", start);
494494
}
@@ -645,7 +645,7 @@ public void export() throws Exception {
645645
if (startLister.currents().size() > 0 || startLister.hasNext()) {
646646
listing(startLister);
647647
} else {
648-
progressMap.remove(startLister.getPrefix());
648+
removeRecordedPrefix(startLister.getPrefix());
649649
startLister.close();
650650
}
651651
} else {

‎src/main/java/com/qiniu/datasource/DatasourceActor.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,11 +94,20 @@ void recordLister(String key, String record) {
9494
procedureLogger.info("{}-|-{}", key, record);
9595
}
9696

97+
void removeRecordedPrefix(String prefix) {
98+
procedureLogger.info("{}-|-", prefix);
99+
progressMap.remove(prefix);
100+
}
101+
97102
void refreshRecordAndStatistics() {
98103
rootLogger.info("finished count: {}.", statistics.get());
99104
if (procedureLogFile.length() > 536870912) { // 超过 512M 就处理一次
100105
try {
101106
breakpointSaver.clear(breakpointFileName);
107+
// StringBuilder record = new StringBuilder("{\"");
108+
// progressMap.forEach((key, m) -> record.append(key).append("\":").append(m).append(",\""));
109+
// record.delete(record.length() - 2, record.length()).append("}");
110+
// breakpointSaver.writeToKey(breakpointFileName, record.toString().replace("\\", "\\\\"), true);
102111
breakpointSaver.writeToKey(breakpointFileName, JsonUtils.toJsonWithoutUrlEscape(progressMap), true);
103112
procedureLogFile.delete();
104113
} catch (IOException e) {
@@ -125,6 +134,10 @@ void endAction() throws IOException {
125134
}
126135
String record = "{}";
127136
if (progressMap.size() > 0) {
137+
// StringBuilder re = new StringBuilder("{\"");
138+
// progressMap.forEach((key, m) -> re.append(key).append("\":").append(m).append(",\""));
139+
// re.delete(re.length() - 2, re.length()).append("}");
140+
// record = re.toString().replace("\\", "\\\\");
128141
record = JsonUtils.toJsonWithoutUrlEscape(progressMap);
129142
breakpointSaver.writeToKey(breakpointFileName, record, true);
130143
breakpointSaver.closeWriters();

‎src/main/java/com/qiniu/entry/CommonParams.java‎

Lines changed: 48 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -715,51 +715,51 @@ private void setPrivateType() throws IOException {
715715
}
716716

717717
private void fromProcedureLog(String logFile, boolean withMarker, boolean withEnd) throws IOException {
718-
String lastLine = FileUtils.lastLineOfFile(logFile);
719-
if (lastLine != null && !"".equals(lastLine)) {
720-
try {
718+
try {
719+
String lastLine = FileUtils.lastLineOfFile(logFile);
720+
if (lastLine != null && !"".equals(lastLine)) {
721721
parseConfigMapFromJson(JsonUtils.toJsonObject(lastLine), withMarker, withEnd);
722-
} catch (Exception e) {
723-
File file = new File(logFile);
724-
FileReader fileReader = new FileReader(file);
725-
BufferedReader bufferedReader = new BufferedReader(fileReader);
726-
int index;
727-
String line;
728-
String value;
729-
Map<String, String> map = new HashMap<>();
730-
while ((line = bufferedReader.readLine()) != null) {
731-
index = line.indexOf("-|-");
732-
if (index < 0) {
733-
try {
734-
parseConfigMapFromJson(JsonUtils.toJsonObject(line), withMarker, withEnd);
735-
return;
736-
} catch (Exception exception) {
737-
exception.printStackTrace();
738-
}
739-
} else {
740-
map.put(line.substring(0, index), line.substring(index));
722+
}
723+
} catch (Exception e) {
724+
File file = new File(logFile);
725+
FileReader fileReader = new FileReader(file);
726+
BufferedReader bufferedReader = new BufferedReader(fileReader);
727+
int index;
728+
String line;
729+
String value;
730+
Map<String, String> map = new HashMap<>();
731+
while ((line = bufferedReader.readLine()) != null) {
732+
index = line.indexOf("-|-");
733+
if (index < 0) {
734+
try {
735+
parseConfigMapFromJson(JsonUtils.toJsonObject(line), withMarker, withEnd);
736+
return;
737+
} catch (Exception exception) {
738+
exception.printStackTrace();
741739
}
740+
} else {
741+
map.put(line.substring(0, index), line.substring(index + 3));
742742
}
743-
Map<String, String> configMap;
744-
for (String key : map.keySet()) {
745-
value = map.get(key);
746-
if (!"".equals(value)) {
747-
try {
748-
configMap = JsonUtils.fromJson(value, map.getClass());
749-
} catch (Exception e1) {
750-
e1.printStackTrace();
751-
continue;
752-
}
753-
pathConfigMap.put(key, configMap);
743+
}
744+
Map<String, String> configMap;
745+
for (String key : map.keySet()) {
746+
value = map.get(key);
747+
if (!"".equals(value)) {
748+
try {
749+
configMap = JsonUtils.fromJson(value, map.getClass());
750+
} catch (Exception e1) {
751+
e1.printStackTrace();
752+
continue;
754753
}
754+
pathConfigMap.put(key, configMap);
755755
}
756-
try {
757-
bufferedReader.close();
758-
fileReader.close();
759-
} catch (IOException ioe) {
760-
bufferedReader = null;
761-
fileReader = null;
762-
}
756+
}
757+
try {
758+
bufferedReader.close();
759+
fileReader.close();
760+
} catch (IOException ioe) {
761+
bufferedReader = null;
762+
fileReader = null;
763763
}
764764
}
765765
}
@@ -1489,6 +1489,10 @@ public void setBucket(String bucket) {
14891489
this.bucket = bucket;
14901490
}
14911491

1492+
public void setLogFilepath(String logFilepath) {
1493+
this.logFilepath = logFilepath;
1494+
}
1495+
14921496
public void setPathConfigMap(Map<String, Map<String, String>> pathConfigMap) {
14931497
this.pathConfigMap = pathConfigMap;
14941498
}
@@ -1689,6 +1693,10 @@ public String getBucket() {
16891693
return bucket;
16901694
}
16911695

1696+
public String getLogFilepath() {
1697+
return logFilepath;
1698+
}
1699+
16921700
public Map<String, Map<String, String>> getPathConfigMap() {
16931701
return pathConfigMap;
16941702
}

‎src/main/java/com/qiniu/util/JsonUtils.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,8 @@ public static String toJsonWithoutUrlEscape(Object srcObject) {
5353
.replace("\"{}\"", "{}")
5454
.replace("\"{\\\"", "{\"")
5555
.replace("\\\"}\"", "\"}")
56-
.replace("\\\":\\\"", "\":\"");
56+
.replace("\\\":\\\"", "\":\"")
57+
.replace("\\\",\\\"", "\",\"");
5758
// .replace("\\\\", "\\");
5859
}
5960

‎src/test/java/com/qiniu/entry/CommonParamsTest.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import com.qiniu.interfaces.IEntryParam;
55
import org.junit.Test;
66

7+
import java.io.FileInputStream;
78
import java.io.IOException;
89
import java.util.Map;
910

@@ -26,4 +27,11 @@ public void test() throws Exception {
2627
Map<String,String> map = commonParams.getIndexMap();
2728
System.out.println(map);
2829
}
30+
31+
@Test
32+
public void testFromProcedureLog() throws Exception {
33+
CommonParams commonParams = new CommonParams();
34+
commonParams.setLogFilepath("/Users/wubingheng/Downloads/procedure0.log.txt");
35+
// commonParams.setPathConfigMap("", null, true, true);
36+
}
2937
}

‎version.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version=8.4.7
1+
version=8.4.8

0 commit comments

Comments
 (0)