Skip to content

Commit e9eae01

Browse files
TomBlockTomBlock
committed
corrected prepStatement for batch updates, added truncate modus
1 parent 6934f3d commit e9eae01

File tree

7 files changed

+176
-42
lines changed

7 files changed

+176
-42
lines changed

‎core/src/main/java/com/bc/fiduceo/db/DbMaintenanceTool.java‎

Lines changed: 52 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import org.apache.commons.cli.HelpFormatter;
99
import org.apache.commons.cli.Option;
1010
import org.apache.commons.cli.Options;
11+
import org.apache.commons.lang.StringUtils;
1112

1213
import java.io.IOException;
1314
import java.io.OutputStream;
@@ -44,12 +45,15 @@ static Options getOptions() {
4445
final Option dryRunOption = new Option("d", "dryrun", false, "Defines 'dryrun' status, i.e. just test the replacement and report problems.");
4546
options.addOption(dryRunOption);
4647

47-
final Option pathOption = new Option("p", "path", true, "Observation path segment to be replaced.");
48+
final Option pathOption = new Option("p", "path", true, "Observation path segment to be replaced or truncated.");
4849
options.addOption(pathOption);
4950

5051
final Option replaceOption = new Option("r", "replace", true, "Observation path segment replacement.");
5152
options.addOption(replaceOption);
5253

54+
final Option truncateOption = new Option("t", "truncate", false, "Command to truncate path segment.");
55+
options.addOption(truncateOption);
56+
5357
final Option segmentsOption = new Option("s", "segments", true, "Number of segments to consider for paths missing the search expression (default: 4)");
5458
options.addOption(segmentsOption);
5559

@@ -75,8 +79,7 @@ void run(CommandLine commandLine) throws IOException, SQLException {
7579

7680
initialize(commandLine);
7781

78-
final String
79-
oldPathSegment = commandLine.getOptionValue("path");
82+
final String oldPathSegment = commandLine.getOptionValue("path");
8083
final String newPathSegment = commandLine.getOptionValue("replace");
8184

8285
final QueryParameter queryParameter = new QueryParameter();
@@ -96,8 +99,15 @@ void run(CommandLine commandLine) throws IOException, SQLException {
9699
accumulator = new PathAccumulator(oldPathSegment, numPathSegments);
97100
executeDryrun(oldPathSegment, queryParameter);
98101
} else {
99-
logger.info("Replacing paths old: " + oldPathSegment + " new: " + newPathSegment);
100-
processUpdate(oldPathSegment, newPathSegment, queryParameter);
102+
final boolean truncate = commandLine.hasOption("truncate");
103+
104+
if (truncate) {
105+
logger.info("Removing path segment: " + oldPathSegment);
106+
processTruncate(oldPathSegment, queryParameter);
107+
} else {
108+
logger.info("Replacing paths - old: " + oldPathSegment + " new: " + newPathSegment);
109+
processUpdate(oldPathSegment, newPathSegment, queryParameter);
110+
}
101111
}
102112
}
103113

@@ -130,6 +140,26 @@ private void executeDryrun(String oldPathSegment, QueryParameter queryParameter)
130140
}
131141
}
132142

143+
private void processTruncate(String oldPathSegment, QueryParameter queryParameter) throws SQLException {
144+
try {
145+
int total_count = 0;
146+
147+
List<SatelliteObservation> satelliteObservations = storage.get(queryParameter);
148+
while (satelliteObservations.size() > 0) {
149+
truncatePaths(oldPathSegment, satelliteObservations);
150+
151+
total_count += satelliteObservations.size();
152+
logger.info("processed " + total_count + " datasets");
153+
154+
final int newOffset = queryParameter.getOffset() + PAGE_SIZE;
155+
queryParameter.setOffset(newOffset);
156+
satelliteObservations = storage.get(queryParameter);
157+
}
158+
} finally {
159+
cleanup();
160+
}
161+
}
162+
133163
private void processUpdate(String oldPathSegment, String newPathSegment, QueryParameter queryParameter) throws SQLException {
134164
try {
135165
int total_count = 0;
@@ -150,6 +180,23 @@ private void processUpdate(String oldPathSegment, String newPathSegment, QueryPa
150180
}
151181
}
152182

183+
private void truncatePaths(String oldPathSegment, List<SatelliteObservation> satelliteObservations) throws SQLException {
184+
AbstractBatch batch = null;
185+
186+
for (final SatelliteObservation observation : satelliteObservations) {
187+
final String oldPath = observation.getDataFilePath().toString();
188+
final int start = oldPath.indexOf(oldPath);
189+
if (start >= 0) {
190+
final String newPath = StringUtils.remove(oldPath, oldPathSegment);
191+
batch = storage.updatePathBatch(observation, newPath, batch);
192+
}
193+
}
194+
195+
if (batch != null) {
196+
storage.commitBatch(batch);
197+
}
198+
}
199+
153200
private void updatePaths(String oldPathSegment, String newPathSegment, List<SatelliteObservation> satelliteObservations) throws SQLException {
154201
AbstractBatch batch = null;
155202

‎core/src/main/java/com/bc/fiduceo/db/H2Driver.java‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -112,17 +112,7 @@ public void insert(SatelliteObservation observation) throws SQLException {
112112
@Override
113113
public AbstractBatch updatePathBatch(SatelliteObservation satelliteObservation, String newPath, AbstractBatch batch) throws SQLException {
114114
if (batch == null) {
115-
final StringBuilder sql = new StringBuilder();
116-
sql.append("UPDATE SATELLITE_OBSERVATION AS obs SET DataFile = ? WHERE obs.stopDate >= '");
117-
sql.append(TimeUtils.format(satelliteObservation.getStartTime(), DATE_PATTERN));
118-
sql.append("' AND obs.startDate <= '");
119-
sql.append(TimeUtils.format(satelliteObservation.getStopTime(), DATE_PATTERN));
120-
sql.append("' AND obs.DataFile = ? AND obs.Version = '");
121-
sql.append(satelliteObservation.getVersion());
122-
sql.append("' AND obs.SensorId = ");
123-
sql.append(getSensorId(satelliteObservation.getSensor().getName()));
124-
125-
final PreparedStatement preparedStatement = connection.prepareStatement(sql.toString());
115+
final PreparedStatement preparedStatement = connection.prepareStatement("UPDATE SATELLITE_OBSERVATION SET DataFile = ? WHERE DataFile = ? ");
126116
batch = new JdbcBatch(preparedStatement);
127117
}
128118

‎core/src/main/java/com/bc/fiduceo/db/MongoBatch.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ class MongoBatch extends AbstractBatch {
1111
private final List<WriteModel<Document>> writeOperations;
1212

1313
public MongoBatch() {
14-
writeOperations = new ArrayList<WriteModel<Document>>();
14+
writeOperations = new ArrayList<>();
1515
}
1616

1717
@Override

‎core/src/main/java/com/bc/fiduceo/db/MongoDbDriver.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -156,10 +156,6 @@ public AbstractBatch updatePathBatch(SatelliteObservation satelliteObservation,
156156
}
157157

158158
final QueryParameter queryParameter = new QueryParameter();
159-
queryParameter.setStartTime(satelliteObservation.getStartTime());
160-
queryParameter.setStopTime(satelliteObservation.getStopTime());
161-
queryParameter.setSensorName(satelliteObservation.getSensor().getName());
162-
queryParameter.setVersion(satelliteObservation.getVersion());
163159
queryParameter.setPath(satelliteObservation.getDataFilePath().toString());
164160

165161
final Document queryDocument = createQueryDocument(queryParameter);

‎core/src/main/java/com/bc/fiduceo/db/PostGISDriver.java‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -157,20 +157,7 @@ public void insert(SatelliteObservation observation) throws SQLException {
157157
@Override
158158
public AbstractBatch updatePathBatch(SatelliteObservation satelliteObservation, String newPath, AbstractBatch batch) throws SQLException {
159159
if (batch == null) {
160-
final StringBuilder sql = new StringBuilder();
161-
sql.append("UPDATE SATELLITE_OBSERVATION AS obs SET DataFile = ? FROM SENSOR AS sen");
162-
sql.append(" WHERE obs.stopDate >= '");
163-
sql.append(satelliteObservation.getStartTime());
164-
sql.append("' AND obs.startDate <= '");
165-
sql.append(satelliteObservation.getStopTime());
166-
sql.append("' AND sen.Name = '");
167-
sql.append(satelliteObservation.getSensor().getName());
168-
sql.append("' AND obs.DataFile = ? ");
169-
sql.append("AND obs.Version = '");
170-
sql.append(satelliteObservation.getVersion());
171-
sql.append("' AND obs.SensorId = sen.ID");
172-
173-
final PreparedStatement preparedStatement = connection.prepareStatement(sql.toString());
160+
final PreparedStatement preparedStatement = connection.prepareStatement("UPDATE SATELLITE_OBSERVATION SET DataFile = ? WHERE DataFile = ?");
174161
batch = new JdbcBatch(preparedStatement);
175162
}
176163

‎core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolIntegrationTest.java‎

Lines changed: 110 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,9 +74,10 @@ public void testInvalidCommandLine() throws ParseException {
7474
" -c,--config <arg> Defines the configuration directory. Defaults to './config'." + ls +
7575
" -d,--dryrun Defines 'dryrun' status, i.e. just test the replacement and report problems." + ls +
7676
" -h,--help Prints the tool usage." + ls +
77-
" -p,--path <arg> Observation path segment to be replaced." + ls +
77+
" -p,--path <arg> Observation path segment to be replaced or truncated." + ls +
7878
" -r,--replace <arg> Observation path segment replacement." + ls +
79-
" -s,--segments <arg> Number of segments to consider for paths missing the search expression (default: 4)" + ls, err.toString());
79+
" -s,--segments <arg> Number of segments to consider for paths missing the search expression (default: 4)" + ls +
80+
" -t,--truncate Command to truncate path segment." + ls, err.toString());
8081
} finally {
8182
System.setOut(_out);
8283
System.setErr(_err);
@@ -141,12 +142,57 @@ public void testCorrectPaths_Postgres_alterSomePaths() throws IOException, Parse
141142
}
142143

143144
@Test
144-
public void testCorrectPaths_H2_alterSomePaths() throws IOException, ParseException, SQLException {
145+
public void testTruncatePaths_MongoDb_alterSomePaths() throws IOException, ParseException, SQLException {
146+
TestUtil.writeDatabaseProperties_MongoDb(configDir);
147+
TestUtil.writeSystemConfig(configDir);
148+
final BasicDataSource dataSource = TestUtil.getDataSource_MongoDb();
149+
150+
runTest_truncatePath(dataSource);
151+
}
152+
153+
@Test
154+
public void testTruncatePaths_Postgres_alterSomePaths() throws IOException, ParseException, SQLException {
155+
TestUtil.writeDatabaseProperties_Postgres(configDir);
156+
TestUtil.writeSystemConfig(configDir);
157+
final BasicDataSource dataSource = TestUtil.getDataSource_Postgres();
158+
159+
runTest_truncatePath(dataSource);
160+
}
161+
162+
@Test
163+
public void testTruncatePaths_H2_alterSomePaths() throws IOException, ParseException, SQLException {
145164
TestUtil.writeDatabaseProperties_H2(configDir);
146165
TestUtil.writeSystemConfig(configDir);
147166
final BasicDataSource dataSource = TestUtil.getDatasource_H2();
148167

149-
runTest_alterSomePaths(dataSource);
168+
runTest_truncatePath(dataSource);
169+
}
170+
171+
@Test
172+
public void testTruncatePaths_MongoDb_innerSegment() throws IOException, ParseException, SQLException {
173+
TestUtil.writeDatabaseProperties_MongoDb(configDir);
174+
TestUtil.writeSystemConfig(configDir);
175+
final BasicDataSource dataSource = TestUtil.getDataSource_MongoDb();
176+
177+
runTest_truncatePath_innerSegment(dataSource);
178+
}
179+
180+
@Test
181+
public void testTruncatePaths_Postgres_innerSegment() throws IOException, ParseException, SQLException {
182+
TestUtil.writeDatabaseProperties_Postgres(configDir);
183+
TestUtil.writeSystemConfig(configDir);
184+
final BasicDataSource dataSource = TestUtil.getDataSource_Postgres();
185+
186+
runTest_truncatePath_innerSegment(dataSource);
187+
}
188+
189+
@Test
190+
public void testTruncatePaths_H2_innerSegment() throws IOException, ParseException, SQLException {
191+
TestUtil.writeDatabaseProperties_H2(configDir);
192+
TestUtil.writeSystemConfig(configDir);
193+
final BasicDataSource dataSource = TestUtil.getDatasource_H2();
194+
195+
runTest_truncatePath_innerSegment(dataSource);
150196
}
151197

152198
@Test
@@ -268,6 +314,66 @@ private void runTest_alterSomePaths(BasicDataSource dataSource) throws SQLExcept
268314
}
269315
}
270316

317+
private void runTest_truncatePath(BasicDataSource dataSource) throws SQLException, ParseException {
318+
final Storage storage = initializeStorage(dataSource);
319+
320+
for (int i = 0; i < 12; i++) {
321+
final SatelliteObservation observation = TestData.createSatelliteObservation(geometryFactory);
322+
final String obsPath = TestUtil.assembleFileSystemPath(new String[]{"archive", "correct", "the_file_number_" + i}, true);
323+
observation.setDataFilePath(obsPath);
324+
storage.insert(observation);
325+
}
326+
327+
try {
328+
final String cutPath = TestUtil.assembleFileSystemPath(new String[]{"archive", "correct"}, true);
329+
330+
final String[] args = new String[]{"-c", configDir.getAbsolutePath(),
331+
"-p", cutPath,
332+
"-t"};
333+
334+
DbMaintenanceToolMain.main(args);
335+
336+
final List<SatelliteObservation> observations = storage.get();
337+
assertEquals(12, observations.size());
338+
for (SatelliteObservation satelliteObservation : observations) {
339+
assertFalse(satelliteObservation.getDataFilePath().toString().contains(cutPath));
340+
}
341+
} finally {
342+
storage.clear();
343+
storage.close();
344+
}
345+
}
346+
347+
private void runTest_truncatePath_innerSegment(BasicDataSource dataSource) throws SQLException, ParseException {
348+
final Storage storage = initializeStorage(dataSource);
349+
350+
for (int i = 0; i < 12; i++) {
351+
final SatelliteObservation observation = TestData.createSatelliteObservation(geometryFactory);
352+
final String obsPath = TestUtil.assembleFileSystemPath(new String[]{"archive", "correct", "the_file_number_" + i}, true);
353+
observation.setDataFilePath(obsPath);
354+
storage.insert(observation);
355+
}
356+
357+
try {
358+
final String cutPath = TestUtil.assembleFileSystemPath(new String[]{"correct"}, true);
359+
360+
final String[] args = new String[]{"-c", configDir.getAbsolutePath(),
361+
"-p", cutPath,
362+
"-t"};
363+
364+
DbMaintenanceToolMain.main(args);
365+
366+
final List<SatelliteObservation> observations = storage.get();
367+
assertEquals(12, observations.size());
368+
for (SatelliteObservation satelliteObservation : observations) {
369+
assertFalse(satelliteObservation.getDataFilePath().toString().contains(cutPath));
370+
}
371+
} finally {
372+
storage.clear();
373+
storage.close();
374+
}
375+
}
376+
271377
private void runTest_dryRun_allOk(BasicDataSource dataSource) throws SQLException, ParseException {
272378
final String sep = System.lineSeparator();
273379
final Storage storage = initializeStorage(dataSource);

‎core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolTest.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ public void testGetOptions() {
3333
assertNotNull(pathOption);
3434
assertEquals("p", pathOption.getOpt());
3535
assertEquals("path", pathOption.getLongOpt());
36-
assertEquals("Observation path segment to be replaced.", pathOption.getDescription());
36+
assertEquals("Observation path segment to be replaced or truncated.", pathOption.getDescription());
3737
assertTrue(pathOption.hasArg());
3838

3939
final Option replaceOption = options.getOption("replace");
@@ -42,6 +42,13 @@ public void testGetOptions() {
4242
assertEquals("replace", replaceOption.getLongOpt());
4343
assertEquals("Observation path segment replacement.", replaceOption.getDescription());
4444
assertTrue(replaceOption.hasArg());
45+
46+
final Option truncateOption = options.getOption("truncate");
47+
assertNotNull(truncateOption);
48+
assertEquals("t", truncateOption.getOpt());
49+
assertEquals("truncate", truncateOption.getLongOpt());
50+
assertEquals("Command to truncate path segment.", truncateOption.getDescription());
51+
assertFalse(truncateOption.hasArg());
4552
}
4653

4754
@Test
@@ -59,8 +66,9 @@ public void testPrintUsageTo() {
5966
" -c,--config <arg> Defines the configuration directory. Defaults to './config'." + ls +
6067
" -d,--dryrun Defines 'dryrun' status, i.e. just test the replacement and report problems." + ls +
6168
" -h,--help Prints the tool usage." + ls +
62-
" -p,--path <arg> Observation path segment to be replaced." + ls +
69+
" -p,--path <arg> Observation path segment to be replaced or truncated." + ls +
6370
" -r,--replace <arg> Observation path segment replacement." + ls +
64-
" -s,--segments <arg> Number of segments to consider for paths missing the search expression (default: 4)" + ls, stream.toString());
71+
" -s,--segments <arg> Number of segments to consider for paths missing the search expression (default: 4)" + ls +
72+
" -t,--truncate Command to truncate path segment." + ls, stream.toString());
6573
}
6674
}

0 commit comments

Comments
 (0)