Skip to content

Commit c4856ee

Browse files
TomBlockTomBlock
committed
update ERA 5 archive file handling
1 parent 9635cf4 commit c4856ee

File tree

11 files changed

+123
-45
lines changed

11 files changed

+123
-45
lines changed

‎FIDUCEO_MMS_Manual_v_1_5.pdf‎

11.2 KB
Binary file not shown.

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5Archive.java‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,29 @@
99
class Era5Archive {
1010

1111
private static final DecimalFormat twoDigitsFormat = new DecimalFormat("00");
12-
private static final DecimalFormat fourDigitsFormat = new DecimalFormat("0000");
13-
private static final String FILE_NAME_BEGIN = "ecmwf-era5_oper_";
1412

1513
private final String rootPath;
14+
private final Era5Collection collection;
1615

17-
Era5Archive(String rootPath) {
16+
Era5Archive(String rootPath, Era5Collection collection) {
1817
this.rootPath = rootPath;
18+
this.collection = collection;
1919
}
2020

21-
static String getFileName(String collection, String variable, String timeString) {
22-
return FILE_NAME_BEGIN + collection + "_" + timeString + "." + variable + ".nc";
21+
String getFileName(String collection, String variable, String timeString) {
22+
return getFileNameBegin() + collection + "_" + timeString + "." + variable + ".nc";
23+
}
24+
25+
private String getFileNameBegin() {
26+
if (collection == Era5Collection.ERA_5) {
27+
return "ecmwf-era5_oper_";
28+
} else if(collection == Era5Collection.ERA_5T) {
29+
return "ecmwf-era5t_oper_";
30+
} else if (collection == Era5Collection.ERA_51) {
31+
return "ecmwf-era51_oper_";
32+
}
33+
34+
throw new IllegalStateException("Unsupported RA5 collection");
2335
}
2436

2537
static String mapVariable(String variable) {
@@ -77,7 +89,7 @@ String get(String variableType, int timeStamp) {
7789
final int cutPoint = variableType.lastIndexOf("_");
7890
final String collection = variableType.substring(0, cutPoint);
7991

80-
String variable = variableType.substring(cutPoint + 1, variableType.length());
92+
String variable = variableType.substring(cutPoint + 1);
8193
variable = mapVariable(variable);
8294

8395
adjustCalendarForForecast(utcCalendar, collection);

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5Collection.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package com.bc.fiduceo.post.plugin.era5;
2+
3+
enum Era5Collection {
4+
ERA_5,
5+
ERA_51,
6+
ERA_5T;
7+
8+
static Era5Collection fromString(String collectionString) {
9+
if (collectionString.equalsIgnoreCase("era5") || collectionString.equalsIgnoreCase("era_5")) {
10+
return ERA_5;
11+
} else if (collectionString.equalsIgnoreCase("era5t") || collectionString.equalsIgnoreCase("era_5t")) {
12+
return ERA_5T;
13+
} else if (collectionString.equalsIgnoreCase("era51") || collectionString.equalsIgnoreCase("era_51")) {
14+
return ERA_51;
15+
}
16+
17+
throw new IllegalArgumentException("Invalid ERA5 collection: " + collectionString);
18+
}
19+
}

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessing.java‎

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -137,11 +137,11 @@ private static InterpolationContext createInterpolationContext_1D(Array lonArray
137137
}
138138

139139
static boolean isValidLon(float lon) {
140-
return lon >= -180.f && lon <= 180.f ;
140+
return lon >= -180.f && lon <= 180.f;
141141
}
142142

143143
static boolean isValidLat(float lat) {
144-
return lat >= -90.f && lat <= 90.f ;
144+
return lat >= -90.f && lat <= 90.f;
145145
}
146146

147147
@Override
@@ -151,45 +151,40 @@ protected void prepare(NetcdfFile reader, NetcdfFileWriter writer) {
151151
throw new RuntimeException("Expected dimension not present in file: " + FiduceoConstants.MATCHUP_COUNT);
152152
}
153153

154-
prepare(writer);
154+
final Era5Collection collection = getEra5Collection(configuration);
155+
writer.addGlobalAttribute("era5-collection", collection.toString());
155156

156157
final SatelliteFieldsConfiguration satFieldsConfig = configuration.getSatelliteFields();
157158
if (satFieldsConfig != null) {
158159
satelliteFields = new SatelliteFields();
159-
satelliteFields.prepare(satFieldsConfig, reader, writer);
160+
satelliteFields.prepare(satFieldsConfig, reader, writer, collection);
160161
}
161162

162163
final MatchupFieldsConfiguration matchupFieldsConfig = configuration.getMatchupFields();
163164
if (matchupFieldsConfig != null) {
164165
matchupFields = new MatchupFields();
165-
matchupFields.prepare(matchupFieldsConfig, reader, writer);
166+
matchupFields.prepare(matchupFieldsConfig, reader, writer, collection);
166167
}
167168
}
168169

169-
private void prepare(NetcdfFileWriter writer) {
170-
String collection = getEra5Collection(configuration);
171-
writer.addGlobalAttribute("era5-collection", collection);
172-
}
173-
174170
// package access for testing only tb 2021-01-12
175-
static String getEra5Collection(Configuration configuration) {
171+
static Era5Collection getEra5Collection(Configuration configuration) {
176172
String collection = configuration.getEra5Collection();
177173
if (StringUtils.isNotNullAndNotEmpty(collection)) {
178-
return collection;
174+
return Era5Collection.fromString(collection);
179175
}
180176

181177
// we need to find the collection in the path-name
182178
final String nwpAuxDir = configuration.getNWPAuxDir();
183179
final String upperCaseAuxDir = nwpAuxDir.toUpperCase();
184-
collection = "UNKNOWN";
185180
if (upperCaseAuxDir.contains("ERA5T") || upperCaseAuxDir.contains("ERA-5T")) {
186-
collection = "ERA-5T";
181+
return Era5Collection.ERA_5T;
187182
} else if (upperCaseAuxDir.contains("ERA51") || upperCaseAuxDir.contains("ERA-51")) {
188-
collection = "ERA-51";
183+
return Era5Collection.ERA_51;
189184
} else if (upperCaseAuxDir.contains("ERA5") || upperCaseAuxDir.contains("ERA-5")) {
190-
collection = "ERA-5";
185+
return Era5Collection.ERA_5;
191186
}
192-
return collection;
187+
throw new IllegalArgumentException("Unable to detect ERA5 collection from datapath. Please configure explicitly.");
193188
}
194189

195190
@Override

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ class MatchupFields extends FieldsProcessor {
2121
private static final int SECS_PER_HOUR = 3600;
2222

2323
private Map<String, TemplateVariable> variables;
24+
private Era5Collection collection;
2425

2526
private static Array createTimeArray(NetcdfFile reader, MatchupFieldsConfiguration matchupConfig, int numTimeSteps, Variable nwpTimeVariable) throws IOException, InvalidRangeException {
2627
final Array timeArray = VariableUtils.readTimeArray(matchupConfig.get_time_variable_name(), reader);
@@ -49,9 +50,11 @@ private static Array createTimeArray(NetcdfFile reader, MatchupFieldsConfigurati
4950
return targetTimeArray;
5051
}
5152

52-
void prepare(MatchupFieldsConfiguration matchupFieldsConfig, NetcdfFile reader, NetcdfFileWriter writer) {
53+
void prepare(MatchupFieldsConfiguration matchupFieldsConfig, NetcdfFile reader, NetcdfFileWriter writer, Era5Collection collection) {
5354
matchupFieldsConfig.verify();
5455

56+
this.collection = collection;
57+
5558
final List<Dimension> dimensions = getDimensions(matchupFieldsConfig, writer, reader);
5659

5760
variables = getVariables(matchupFieldsConfig);
@@ -65,7 +68,7 @@ void prepare(MatchupFieldsConfiguration matchupFieldsConfig, NetcdfFile reader,
6568
}
6669

6770
void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) throws IOException, InvalidRangeException {
68-
final Era5Archive era5Archive = new Era5Archive(config.getNWPAuxDir());
71+
final Era5Archive era5Archive = new Era5Archive(config.getNWPAuxDir(), collection);
6972
final MatchupFieldsConfiguration matchupConfig = config.getMatchupFields();
7073

7174
// allocate cache large enough to hold the time-series for one Era-5 variable

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ class SatelliteFields extends FieldsProcessor {
2323
private List<Dimension> dimension2d;
2424
private List<Dimension> dimension3d;
2525
private Map<String, TemplateVariable> variables;
26+
private Era5Collection collection;
2627

2728
static Array readSubset(int numLayers, Rectangle era5RasterPosition, VariableCache.CacheEntry cacheEntry) throws IOException, InvalidRangeException {
2829
Array subset;
@@ -161,10 +162,12 @@ private static Array readVariableData(int numLayers, Rectangle era5RasterPositio
161162
return subset;
162163
}
163164

164-
void prepare(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, NetcdfFileWriter writer) {
165+
void prepare(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, NetcdfFileWriter writer, Era5Collection collection) {
165166
satFieldsConfig.verify();
166167
setDimensions(satFieldsConfig, writer, reader);
167168

169+
this.collection = collection;
170+
168171
variables = getVariables(satFieldsConfig);
169172
final Collection<TemplateVariable> values = variables.values();
170173
for (TemplateVariable template : values) {
@@ -180,7 +183,7 @@ void prepare(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, Ne
180183
void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) throws IOException, InvalidRangeException {
181184
final SatelliteFieldsConfiguration satFieldsConfig = config.getSatelliteFields();
182185
final int numLayers = satFieldsConfig.get_z_dim();
183-
final Era5Archive era5Archive = new Era5Archive(config.getNWPAuxDir());
186+
final Era5Archive era5Archive = new Era5Archive(config.getNWPAuxDir(), collection);
184187
final VariableCache variableCache = new VariableCache(era5Archive, 52); // 4 * 13 variables tb 2020-11-25
185188

186189
try {

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolIntegrationTest_Era5.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ public void testAddEra5Variables_mmd15() throws IOException, InvalidRangeExcepti
7575
assertTrue(targetFile.isFile());
7676

7777
try (NetcdfFile mmd = NetcdfFiles.open(targetFile.getAbsolutePath())) {
78-
NCTestUtils.assertGlobalAttribute(mmd, "era5-collection", "ERA-5");
78+
NCTestUtils.assertGlobalAttribute(mmd, "era5-collection", "ERA_5");
7979

8080
Variable variable = NCTestUtils.getVariable("amsre\\.Geostationary_Reflection_Latitude", mmd, false);
8181
NCTestUtils.assert3DValueDouble(0, 0, 0, 4105, variable);
@@ -167,7 +167,7 @@ public void testAddEra5Variables_coo1() throws IOException, InvalidRangeExceptio
167167
assertTrue(targetFile.isFile());
168168

169169
try (NetcdfFile mmd = NetcdfFiles.open(targetFile.getAbsolutePath())) {
170-
NCTestUtils.assertGlobalAttribute(mmd, "era5-collection", "ERA-5");
170+
NCTestUtils.assertGlobalAttribute(mmd, "era5-collection", "ERA_5");
171171

172172
Variable variable = NCTestUtils.getVariable("avhrr-frac-ma_delta_azimuth", mmd, false);
173173
NCTestUtils.assert3DValueDouble(0, 0, 0, 11.972550392150879, variable);

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5ArchiveTest.java‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ public class Era5ArchiveTest {
1616
private static final String SEP = File.separator;
1717

1818
@Test
19-
public void testConstructAndGet() throws IOException {
20-
final Era5Archive era5Archive = new Era5Archive("archive" + SEP + "era5");
19+
public void testConstructAndGet() {
20+
final Era5Archive era5Archive = new Era5Archive("archive" + SEP + "era5", Era5Collection.ERA_5);
2121

2222
// Friday, 30. May 2008 11:00:00
2323
// 1212145200
@@ -40,8 +40,21 @@ public void testConstructAndGet() throws IOException {
4040
}
4141

4242
@Test
43-
public void testGetFileName() {
44-
assertEquals("ecmwf-era5_oper_an_ml_201108231900.q.nc", Era5Archive.getFileName("an_ml", "q", "201108231900"));
43+
public void testGetFileName_era5() {
44+
final Era5Archive archive = new Era5Archive("whatever", Era5Collection.ERA_5);
45+
assertEquals("ecmwf-era5_oper_an_ml_201108231900.q.nc", archive.getFileName("an_ml", "q", "201108231900"));
46+
}
47+
48+
@Test
49+
public void testGetFileName_era5t() {
50+
final Era5Archive archive = new Era5Archive("whatever", Era5Collection.ERA_5T);
51+
assertEquals("ecmwf-era5t_oper_an_ml_201108232000.q.nc", archive.getFileName("an_ml", "q", "201108232000"));
52+
}
53+
54+
@Test
55+
public void testGetFileName_era51() {
56+
final Era5Archive archive = new Era5Archive("whatever", Era5Collection.ERA_51);
57+
assertEquals("ecmwf-era51_oper_an_fc_201108232000.q.nc", archive.getFileName("an_fc", "q", "201108232000"));
4558
}
4659

4760
@Test

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5CollectionTest.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package com.bc.fiduceo.post.plugin.era5;
2+
3+
import org.junit.Test;
4+
5+
import static org.junit.Assert.assertEquals;
6+
import static org.junit.Assert.fail;
7+
8+
public class Era5CollectionTest {
9+
10+
@Test
11+
public void testFromString() {
12+
assertEquals(Era5Collection.ERA_5, Era5Collection.fromString("ERA5"));
13+
assertEquals(Era5Collection.ERA_5, Era5Collection.fromString("ERA_5"));
14+
15+
assertEquals(Era5Collection.ERA_5T, Era5Collection.fromString("era5t"));
16+
assertEquals(Era5Collection.ERA_5T, Era5Collection.fromString("era_5t"));
17+
18+
assertEquals(Era5Collection.ERA_51, Era5Collection.fromString("ERA51"));
19+
assertEquals(Era5Collection.ERA_51, Era5Collection.fromString("era_51"));
20+
}
21+
22+
@Test
23+
public void testFromString_invalid() {
24+
try {
25+
Era5Collection.fromString("Nasenmann");
26+
fail("IllegalArgumentException expected");
27+
} catch (IllegalArgumentException expected) {
28+
}
29+
}
30+
}

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingTest.java‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -70,42 +70,45 @@ public void testGetInterpolationContext_singlePixel() {
7070
public void testGetEra5Collection_fromConfig() {
7171
final Configuration configuration = new Configuration();
7272
configuration.setNWPAuxDir("/yamas/strange/path");
73-
configuration.setEra5Collection("Hamasuki");
73+
configuration.setEra5Collection("era_5t");
7474

75-
final String era5Collection = Era5PostProcessing.getEra5Collection(configuration);
76-
assertEquals("Hamasuki", era5Collection);
75+
final Era5Collection era5Collection = Era5PostProcessing.getEra5Collection(configuration);
76+
assertEquals(Era5Collection.ERA_5T, era5Collection);
7777
}
7878

7979
@Test
8080
public void testGetEra5Collection_fromPath_unknown() {
8181
final Configuration configuration = new Configuration();
8282
configuration.setNWPAuxDir("/yamas/strange/path");
8383

84-
final String era5Collection = Era5PostProcessing.getEra5Collection(configuration);
85-
assertEquals("UNKNOWN", era5Collection);
84+
try {
85+
Era5PostProcessing.getEra5Collection(configuration);
86+
fail("IllegalArgumentException expected");
87+
} catch (IllegalArgumentException expected) {
88+
}
8689
}
8790

8891
@Test
8992
public void testGetEra5Collection_fromPath() {
9093
final Configuration configuration = new Configuration();
9194

9295
configuration.setNWPAuxDir("/data/era5");
93-
assertEquals("ERA-5", Era5PostProcessing.getEra5Collection(configuration));
96+
assertEquals(Era5Collection.ERA_5, Era5PostProcessing.getEra5Collection(configuration));
9497

9598
configuration.setNWPAuxDir("/data/era-5");
96-
assertEquals("ERA-5", Era5PostProcessing.getEra5Collection(configuration));
99+
assertEquals(Era5Collection.ERA_5, Era5PostProcessing.getEra5Collection(configuration));
97100

98101
configuration.setNWPAuxDir("/data/ERA51");
99-
assertEquals("ERA-51", Era5PostProcessing.getEra5Collection(configuration));
102+
assertEquals(Era5Collection.ERA_51, Era5PostProcessing.getEra5Collection(configuration));
100103

101104
configuration.setNWPAuxDir("/data/era-51");
102-
assertEquals("ERA-51", Era5PostProcessing.getEra5Collection(configuration));
105+
assertEquals(Era5Collection.ERA_51, Era5PostProcessing.getEra5Collection(configuration));
103106

104107
configuration.setNWPAuxDir("/data/era5t");
105-
assertEquals("ERA-5T", Era5PostProcessing.getEra5Collection(configuration));
108+
assertEquals(Era5Collection.ERA_5T, Era5PostProcessing.getEra5Collection(configuration));
106109

107110
configuration.setNWPAuxDir("/data/ERA-5T");
108-
assertEquals("ERA-5T", Era5PostProcessing.getEra5Collection(configuration));
111+
assertEquals(Era5Collection.ERA_5T, Era5PostProcessing.getEra5Collection(configuration));
109112
}
110113

111114
@Test

0 commit comments

Comments
 (0)