Skip to content

Commit 9163733

Browse files
TomBlockTomBlock
authored
Merge pull request #12 from bcdev/se_user_defined_global_attributes
User configurable global attributes
2 parents d655929 + db4035c commit 9163733

File tree

6 files changed

+153
-10
lines changed

6 files changed

+153
-10
lines changed

‎CHANGES.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
### Updates from version 1.5.7 to 1.5.8
2+
* added functionality to configure several global attributes before computing MMD files.
23
* added support for NOAA NDBC insitu data
34
* added support for Windsat Coriolis data
45
* added conda environment

‎MMS_Manual_v_1_5.pdf‎

-1.07 MB
Binary file not shown.

‎matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/AbstractMmdWriter.java‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@
5151

5252
abstract class AbstractMmdWriter implements MmdWriter, Target {
5353

54+
static final String GLOBAL_ATTR_TITLE = "title";
55+
static final String GLOBAL_ATTR_INSTITUTION = "institution";
56+
static final String GLOBAL_ATTR_CONTACT = "contact";
57+
static final String GLOBAL_ATTR_LICENSE = "license";
58+
5459
private final Logger logger;
5560
private final Map<String, Array> dataCacheMap;
5661
private final Map<String, Variable> variableMap;
@@ -291,9 +296,9 @@ void initializeNetcdfFile(Path mmdFile, UseCaseConfig useCaseConfig, List<IOVari
291296
netcdfFileWriter.addDimension(null, dimension.getShortName(), dimension.getLength());
292297
}
293298
final Variable variable = netcdfFileWriter.addVariable(null,
294-
ioVariable.getTargetVariableName(),
295-
DataType.getType(ioVariable.getDataType()),
296-
ioVariable.getDimensionNames());
299+
ioVariable.getTargetVariableName(),
300+
DataType.getType(ioVariable.getDataType()),
301+
ioVariable.getDimensionNames());
297302
final List<Attribute> attributes = ioVariable.getAttributes();
298303
for (Attribute attribute : attributes) {
299304
variable.addAttribute(attribute);
@@ -366,10 +371,17 @@ private Variable getVariable(String variableName) {
366371
}
367372

368373
private void createGlobalAttributes() {
369-
addGlobalAttribute("title", "FIDUCEO multi-sensor match-up dataset (MMD)");
370-
addGlobalAttribute("institution", "Brockmann Consult GmbH");
371-
addGlobalAttribute("contact", "Tom Block ([email protected])");
372-
addGlobalAttribute("license", "This dataset is released for use under CC-BY licence and was developed in the EC FIDUCEO project \"Fidelity and Uncertainty in Climate Data Records from Earth Observations\". Grant Agreement: 638822.");
374+
final Map<String, String> ga = writerConfig.getGlobalAttributes();
375+
376+
final String title = ga.getOrDefault(GLOBAL_ATTR_TITLE, "SCEPS multi-sensor match-up dataset (MMD)");
377+
final String institution = ga.getOrDefault(GLOBAL_ATTR_INSTITUTION, "Brockmann Consult GmbH");
378+
final String contact = ga.getOrDefault(GLOBAL_ATTR_CONTACT, "Tom Block ([email protected])");
379+
final String licence = ga.getOrDefault(GLOBAL_ATTR_LICENSE, "ESA Data Policy: free and open access.");
380+
381+
addGlobalAttribute(GLOBAL_ATTR_TITLE, title);
382+
addGlobalAttribute(GLOBAL_ATTR_INSTITUTION, institution);
383+
addGlobalAttribute(GLOBAL_ATTR_CONTACT, contact);
384+
addGlobalAttribute(GLOBAL_ATTR_LICENSE, licence);
373385
addGlobalAttribute("creation_date", TimeUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss"));
374386
addGlobalAttribute("software_version", FiduceoConstants.VERSION);
375387
}

‎matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/MmdWriterConfig.java‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ public class MmdWriterConfig {
4141
private static final String CACHE_SIZE_TAG = "cache-size";
4242
private static final String NETCDF_FORMAT_TAG = "netcdf-format";
4343
private static final String READER_CACHE_SIZE_TAG = "reader-cache-size";
44+
private static final String GLOBAL_ATTRIBUTES_TAG = "global-attributes";
4445
private static final String VARIABLES_CONFIGURATION_TAG = "variables-configuration";
4546
private static final String SENSOR_RENAME_TAG = "sensor-rename";
4647
private static final String SEPARATOR = "separator";
@@ -61,12 +62,14 @@ public class MmdWriterConfig {
6162
private NetcdfType netcdfFormat;
6263
private VariablesConfiguration variablesConfiguration;
6364
private int readerCacheSize;
65+
private Map<String, String> globalAttributes;
6466

6567
MmdWriterConfig() {
6668
cacheSize = 2048;
6769
netcdfFormat = NetcdfType.N4;
6870
variablesConfiguration = new VariablesConfiguration();
6971
readerCacheSize = 6;
72+
globalAttributes = new HashMap<>();
7073
}
7174

7275
private MmdWriterConfig(Document document) {
@@ -96,6 +99,10 @@ public VariablesConfiguration getVariablesConfiguration() {
9699
return variablesConfiguration;
97100
}
98101

102+
public Map<String, String> getGlobalAttributes() {
103+
return Collections.unmodifiableMap(globalAttributes);
104+
}
105+
99106
int getCacheSize() {
100107
return cacheSize;
101108
}
@@ -151,6 +158,16 @@ private void init(Document document) {
151158
setReaderCacheSize(Integer.valueOf(readerCacheValue));
152159
}
153160

161+
final Element globalAttributesElement = rootElement.getChild(GLOBAL_ATTRIBUTES_TAG);
162+
if (globalAttributesElement != null) {
163+
final List<Element> elements = globalAttributesElement.getChildren("attribute");
164+
for (Element element : elements) {
165+
final String attName = element.getAttributeValue("name");
166+
final String attValue = element.getAttributeValue("value");
167+
globalAttributes.put(attName, attValue);
168+
}
169+
}
170+
154171
final Element variablesConfigurationElement = rootElement.getChild(VARIABLES_CONFIGURATION_TAG);
155172
if (variablesConfigurationElement != null) {
156173
addSensorRenames(variablesConfigurationElement);
@@ -250,7 +267,7 @@ private void addWriteScaledConfig(Element sensorElement) {
250267
final List<Element> writeScaledElements = sensorElement.getChildren("writeScaled");
251268
for (final Element scaledVarElement : writeScaledElements) {
252269
final String sourceName = getAttributeString(SOURCE_NAME_ATTRIBUTE, scaledVarElement);
253-
for (final String sensorName: sensorNames) {
270+
for (final String sensorName : sensorNames) {
254271
variablesConfiguration.addWriteScaled(sensorName, sourceName);
255272
}
256273
}

‎matchup-tool/src/test/java/com/bc/fiduceo/matchup/writer/MmdWriterConfigTest.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@
2121
package com.bc.fiduceo.matchup.writer;
2222

2323

24+
import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_TITLE;
25+
import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_INSTITUTION;
26+
import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_CONTACT;
27+
import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_LICENSE;
2428
import static com.bc.fiduceo.matchup.writer.MmdWriterFactory.NetcdfType.N3;
2529
import static com.bc.fiduceo.matchup.writer.MmdWriterFactory.NetcdfType.N4;
2630
import static org.junit.Assert.*;
@@ -161,6 +165,49 @@ public void testLoad_emptyVariablesConfiguration() {
161165
assertNotNull(variablesConfiguration);
162166
}
163167

168+
@Test
169+
public void testLoad_emptyGlobalAttributesConfiguration() {
170+
//preparation
171+
final String configXml = "<mmd-writer-config>" +
172+
" <global-attributes/>" +
173+
"</mmd-writer-config>";
174+
final ByteArrayInputStream inputStream = new ByteArrayInputStream(configXml.getBytes());
175+
176+
//execution
177+
final MmdWriterConfig loadedConfig = MmdWriterConfig.load(inputStream);
178+
final Map variablesConfiguration = loadedConfig.getGlobalAttributes();
179+
180+
//verification
181+
assertNotNull(variablesConfiguration);
182+
assertEquals(0, variablesConfiguration.size());
183+
}
184+
185+
@Test
186+
public void testLoad_fullGlobalAttributesConfiguration() {
187+
//preparation
188+
final String configXml = "<mmd-writer-config>" +
189+
" <global-attributes>" +
190+
" <attribute name=\"" + GLOBAL_ATTR_TITLE + "\" value=\"Ein\" />" +
191+
" <attribute name=\"" + GLOBAL_ATTR_INSTITUTION + "\" value=\"wunder\" />" +
192+
" <attribute name=\"" + GLOBAL_ATTR_CONTACT + "\" value=\"schöner\" />" +
193+
" <attribute name=\"" + GLOBAL_ATTR_LICENSE + "\" value=\"Tag\" />" +
194+
" </global-attributes>" +
195+
"</mmd-writer-config>";
196+
final ByteArrayInputStream inputStream = new ByteArrayInputStream(configXml.getBytes());
197+
198+
//execution
199+
final MmdWriterConfig loadedConfig = MmdWriterConfig.load(inputStream);
200+
final Map globAttr = loadedConfig.getGlobalAttributes();
201+
202+
//verification
203+
assertNotNull(globAttr);
204+
assertEquals(4, globAttr.size());
205+
assertEquals("Ein", globAttr.get(GLOBAL_ATTR_TITLE) );
206+
assertEquals("wunder", globAttr.get(GLOBAL_ATTR_INSTITUTION) );
207+
assertEquals("schöner", globAttr.get(GLOBAL_ATTR_CONTACT) );
208+
assertEquals("Tag", globAttr.get(GLOBAL_ATTR_LICENSE) );
209+
}
210+
164211
@Test
165212
public void testLoad_AttributeRename() {
166213
final String configXml = "<mmd-writer-config>" +

‎matchup-tool/src/test/java/com/bc/fiduceo/matchup/writer/MmdWriter_IO_Test.java‎

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import ucar.nc2.Variable;
4646
import ucar.nc2.iosp.netcdf3.N3iosp;
4747

48+
import java.io.ByteArrayInputStream;
4849
import java.io.File;
4950
import java.io.IOException;
5051
import java.io.PrintWriter;
@@ -57,6 +58,10 @@
5758
import java.util.Date;
5859
import java.util.List;
5960

61+
import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_TITLE;
62+
import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_INSTITUTION;
63+
import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_CONTACT;
64+
import static com.bc.fiduceo.matchup.writer.AbstractMmdWriter.GLOBAL_ATTR_LICENSE;
6065
import static org.hamcrest.Matchers.equalToIgnoringWhiteSpace;
6166
import static org.junit.Assert.*;
6267

@@ -211,10 +216,10 @@ public void testCreate() throws IOException, InvalidRangeException {
211216

212217
assertEquals(9, mmd.getGlobalAttributes().size());
213218

214-
assertGlobalAttribute("title", "FIDUCEO multi-sensor match-up dataset (MMD)", mmd);
219+
assertGlobalAttribute("title", "SCEPS multi-sensor match-up dataset (MMD)", mmd);
215220
assertGlobalAttribute("institution", "Brockmann Consult GmbH", mmd);
216221
assertGlobalAttribute("contact", "Tom Block ([email protected])", mmd);
217-
assertGlobalAttribute("license", "This dataset is released for use under CC-BY licence and was developed in the EC FIDUCEO project \"Fidelity and Uncertainty in Climate Data Records from Earth Observations\". Grant Agreement: 638822.", mmd);
222+
assertGlobalAttribute("license", "ESA Data Policy: free and open access.", mmd);
218223
assertGlobalDateAttribute("creation_date", TimeUtils.createNow(), mmd);
219224
assertGlobalAttribute("software_version", FiduceoConstants.VERSION, mmd);
220225
assertGlobalAttribute("sensor-names", "avhrr-n11,avhrr-n12", mmd);
@@ -298,6 +303,67 @@ public void testCreate() throws IOException, InvalidRangeException {
298303
}
299304
}
300305

306+
@Test
307+
public void testCreate_withUserDefinedGlobalAttributes() throws IOException, InvalidRangeException {
308+
//preparation
309+
final String configXml = "<mmd-writer-config>" +
310+
" <global-attributes>" +
311+
" <attribute name=\"" + GLOBAL_ATTR_TITLE + "\" value=\"Ein\" />" +
312+
" <attribute name=\"" + GLOBAL_ATTR_INSTITUTION + "\" value=\"wunder\" />" +
313+
" <attribute name=\"" + GLOBAL_ATTR_CONTACT + "\" value=\"schöner\" />" +
314+
" <attribute name=\"" + GLOBAL_ATTR_LICENSE + "\" value=\"Tag\" />" +
315+
" </global-attributes>" +
316+
"</mmd-writer-config>";
317+
final ByteArrayInputStream inputStream = new ByteArrayInputStream(configXml.getBytes());
318+
319+
final MmdWriterConfig loadedConfig = MmdWriterConfig.load(inputStream);
320+
final MmdWriterNC3 mmdWriter = new MmdWriterNC3(loadedConfig);
321+
322+
final Sensor primarySensor = new Sensor("avhrr-n11");
323+
primarySensor.setPrimary(true);
324+
325+
final UseCaseConfig useCaseConfig = UseCaseConfigBuilder
326+
.build("useCaseName")
327+
.withDimensions(Arrays.asList(
328+
new Dimension("avhrr-n11", 5, 7),
329+
new Dimension("avhrr-n12", 3, 5)))
330+
.withSensors(Arrays.asList(
331+
primarySensor,
332+
new Sensor("avhrr-n12")))
333+
.createConfig();
334+
335+
final Path mmdFile = Paths.get(testDir.toURI()).resolve("test_mmd.nc");
336+
337+
//execution
338+
try {
339+
mmdWriter.initializeNetcdfFile(mmdFile, useCaseConfig, new ArrayList<IOVariable>(), 123);
340+
} finally {
341+
mmdWriter.close();
342+
}
343+
344+
//verification
345+
assertTrue(Files.isRegularFile(mmdFile));
346+
347+
NetcdfFile mmd = null;
348+
try {
349+
mmd = NetcdfFile.open(mmdFile.toString());
350+
351+
assertEquals(9, mmd.getGlobalAttributes().size());
352+
353+
assertGlobalAttribute("title", "Ein", mmd);
354+
assertGlobalAttribute("institution", "wunder", mmd);
355+
assertGlobalAttribute("contact", "schöner", mmd);
356+
assertGlobalAttribute("license", "Tag", mmd);
357+
assertGlobalDateAttribute("creation_date", TimeUtils.createNow(), mmd);
358+
assertGlobalAttribute("software_version", FiduceoConstants.VERSION, mmd);
359+
360+
} finally {
361+
if (mmd != null) {
362+
mmd.close();
363+
}
364+
}
365+
}
366+
301367
@Test
302368
public void testWrite_usecase02_AVHRR_NC3() throws IOException, InvalidRangeException {
303369
final MmdWriter mmdWriter = new MmdWriterNC3(writerConfig);

0 commit comments

Comments
 (0)