Skip to content

Commit 10b8a58

Browse files
cboehmecboehme
committed
Merge remote-tracking branch 'origin/master' into quantors-for-metamorph
Updated equalsFilter to support the new if statement Conflicts: src/test/java/org/culturegraph/mf/morph/collectors/CollectorTest.java
2 parents 0530d6a + c32b083 commit 10b8a58

File tree

14 files changed

+999
-31
lines changed

14 files changed

+999
-31
lines changed

‎src/main/java/org/culturegraph/mf/Flux.java

Lines changed: 63 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
import java.io.File;
1919
import java.io.FilenameFilter;
2020
import java.io.IOException;
21+
import java.net.MalformedURLException;
22+
import java.net.URISyntaxException;
2123
import java.net.URL;
2224
import java.net.URLClassLoader;
2325
import java.util.HashMap;
@@ -34,13 +36,19 @@
3436

3537
/**
3638
* @author Markus Michael Geipel
37-
*
39+
*
3840
*/
3941
public final class Flux {
4042
public static final String MODULES_DIR = "modules";
43+
44+
private static final String JAR_FILE_EXTENSION = ".jar";
45+
private static final String CLASS_FILE_EXTENSION = ".class";
46+
4147
private static final Pattern VAR_PATTERN = Pattern.compile("([^=]*)=(.*)");
4248
private static final String SCRIPT_HOME = "FLUX_DIR";
4349

50+
private static final String MODULES_DIR_ARG = "-modules=";
51+
4452
private Flux() {
4553
// no instances
4654
}
@@ -49,34 +57,27 @@ private Flux() {
4957
* @param args
5058
* @throws IOException
5159
* @throws RecognitionException
60+
* @throws URISyntaxException
5261
*/
5362
public static void main(final String[] args) throws IOException, RecognitionException {
5463

55-
final File modulesDir = new File(MODULES_DIR);
56-
if (modulesDir.exists()) {
57-
final FilenameFilter filter = new FilenameFilter() {
58-
@Override
59-
public boolean accept(final File dir, final String name) {
60-
return name.endsWith(".jar") || name.endsWith(".class");
61-
}
62-
};
63-
final List<URL> moduleURLs = new LinkedList<URL>();
64-
for (File file : modulesDir.listFiles(filter)) {
65-
moduleURLs.add(file.getAbsoluteFile().toURI().toURL());
66-
}
67-
final URLClassLoader moduleLoader = new URLClassLoader(moduleURLs.toArray(new URL[0]), Thread
68-
.currentThread().getContextClassLoader());
69-
Thread.currentThread().setContextClassLoader(moduleLoader);
64+
loadModules(getModulesDir(args));
65+
66+
final int fileArg;
67+
if (args.length > 0 && args[0].startsWith(MODULES_DIR_ARG)) {
68+
fileArg = 1;
69+
} else {
70+
fileArg = 0;
7071
}
7172

72-
if (args.length < 1) {
73+
if (args.length < (fileArg + 1)) {
7374
FluxProgramm.printHelp(System.out);
7475
System.exit(2);
7576
} else {
7677

77-
final File fluxFile = new File(args[0]);
78+
final File fluxFile = new File(args[fileArg]);
7879
if (!fluxFile.exists()) {
79-
System.err.println("File not found: " + args[0]);
80+
System.err.println("File not found: " + args[fileArg]);
8081
System.exit(1);
8182
return;
8283
}
@@ -85,7 +86,10 @@ public boolean accept(final File dir, final String name) {
8586
final Map<String, String> vars = new HashMap<String, String>();
8687
vars.put(SCRIPT_HOME, fluxFile.getAbsoluteFile().getParent() + System.getProperty("file.separator"));
8788

88-
for (int i = 1; i < args.length; ++i) {
89+
for (int i = fileArg + 1; i < args.length; ++i) {
90+
if (args[i].startsWith(MODULES_DIR_ARG)) {
91+
continue;
92+
}
8993
final Matcher matcher = VAR_PATTERN.matcher(args[i]);
9094
if (!matcher.find()) {
9195
FluxProgramm.printHelp(System.err);
@@ -96,9 +100,48 @@ public boolean accept(final File dir, final String name) {
96100

97101
// run parser and builder
98102
FluxCompiler.compile(ResourceUtil.getStream(fluxFile), vars).start();
103+
}
104+
}
99105

106+
private static File getModulesDir(final String[] args) {
107+
File modulesDir = new File(MODULES_DIR);
108+
109+
File programDir = null;
110+
try {
111+
programDir = new File(Flux.class.getProtectionDomain().getCodeSource().getLocation().toURI());
112+
} catch (final URISyntaxException e) {
113+
// Ignore the programDir, if it is not available
114+
}
115+
if (programDir != null) {
116+
if (programDir.getName().endsWith(JAR_FILE_EXTENSION)) {
117+
programDir = programDir.getParentFile();
118+
}
119+
modulesDir = new File(programDir, MODULES_DIR);
120+
}
121+
122+
if (args.length > 0 && args[0].startsWith(MODULES_DIR_ARG)) {
123+
modulesDir = new File(args[0].substring(MODULES_DIR_ARG.length()));
100124
}
125+
126+
return modulesDir;
101127
}
102128

129+
private static void loadModules(final File modulesDir) throws MalformedURLException {
130+
if (modulesDir.exists()) {
131+
final FilenameFilter filter = new FilenameFilter() {
132+
@Override
133+
public boolean accept(final File dir, final String name) {
134+
return name.endsWith(JAR_FILE_EXTENSION) || name.endsWith(CLASS_FILE_EXTENSION);
135+
}
136+
};
137+
final List<URL> moduleURLs = new LinkedList<URL>();
138+
for (final File file : modulesDir.listFiles(filter)) {
139+
moduleURLs.add(file.getAbsoluteFile().toURI().toURL());
140+
}
141+
final URLClassLoader moduleLoader = new URLClassLoader(moduleURLs.toArray(new URL[0]), Thread
142+
.currentThread().getContextClassLoader());
143+
Thread.currentThread().setContextClassLoader(moduleLoader);
144+
}
145+
}
103146

104147
}

‎src/main/java/org/culturegraph/mf/morph/Data.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,8 @@ public void setName(final String name) {
4242
this.name = name;
4343
}
4444

45+
@Override
46+
public String toString() {
47+
return name;
48+
}
4549
}

‎src/main/java/org/culturegraph/mf/morph/collectors/EqualsFilter.java

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
* Copyright 2013 Deutsche Nationalbibliothek
3+
*
4+
* Licensed under the Apache License, Version 2.0 the "License";
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.culturegraph.mf.morph.collectors;
17+
18+
import java.util.HashMap;
19+
import java.util.HashSet;
20+
import java.util.Map;
21+
import java.util.Set;
22+
23+
import org.culturegraph.mf.morph.Metamorph;
24+
import org.culturegraph.mf.morph.NamedValueSource;
25+
import org.culturegraph.mf.util.StringUtil;
26+
27+
28+
29+
/**
30+
* Corresponds to the <code>&lt;equalsFilter-literal&gt;</code> tag. Emits data if values in variables are all equal.
31+
*
32+
* @author Thomas Haidlas
33+
*/
34+
public final class EqualsFilter extends AbstractFlushingCollect{
35+
private final Map<String, String> variables = new HashMap<String, String>();
36+
private final Set<NamedValueSource> sources = new HashSet<NamedValueSource>();
37+
private final Set<NamedValueSource> sourcesLeft = new HashSet<NamedValueSource>();
38+
private boolean isEqual = true;
39+
40+
public EqualsFilter(final Metamorph metamorph) {
41+
super(metamorph);
42+
setNamedValueReceiver(metamorph);
43+
this.isEqual = true;
44+
}
45+
46+
@Override
47+
protected void emit() {
48+
final String name = StringUtil.format(getName(), this.variables);
49+
final String value = StringUtil.format(getValue(), this.variables);
50+
if(this.isEqual){
51+
getNamedValueReceiver().receive(name, value, this, getRecordCount(), getEntityCount());
52+
}
53+
}
54+
55+
56+
@Override
57+
protected boolean isComplete() {
58+
return this.sourcesLeft.isEmpty();
59+
}
60+
61+
@Override
62+
protected void receive(final String name, final String value, final NamedValueSource source) {
63+
if(this.variables.size() > 0 && !this.variables.containsValue(value)){
64+
this.isEqual = false;
65+
}
66+
this.variables.put(name, value);
67+
this.sourcesLeft.remove(source);
68+
}
69+
70+
71+
@Override
72+
public void onNamedValueSourceAdded(final NamedValueSource namedValueSource) {
73+
this.sources.add(namedValueSource);
74+
this.sourcesLeft.add(namedValueSource);
75+
}
76+
77+
@Override
78+
protected void clear() {
79+
this.sourcesLeft.addAll(this.sources);
80+
this.variables.clear();
81+
this.isEqual = true;
82+
}
83+
84+
}

‎src/main/java/org/culturegraph/mf/stream/converter/RecordReader.java

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Copyright 2013 Christoph Böhme
3+
*
4+
* Licensed under the Apache License, Version 2.0 the "License";
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.culturegraph.mf.stream.converter;
17+
18+
import java.io.IOException;
19+
import java.io.Reader;
20+
21+
import org.culturegraph.mf.exceptions.MetafactureException;
22+
import org.culturegraph.mf.framework.DefaultObjectPipe;
23+
import org.culturegraph.mf.framework.ObjectReceiver;
24+
import org.culturegraph.mf.framework.annotations.Description;
25+
import org.culturegraph.mf.framework.annotations.In;
26+
import org.culturegraph.mf.framework.annotations.Out;
27+
28+
/**
29+
* <p>Reads data from a {@code Reader} and splits it into individual
30+
* records.</p>
31+
*
32+
* <p>The default separator is the global separator character (0x001d).
33+
* Empty records are skipped by default.</p>
34+
*
35+
* @author Christoph Böhme
36+
*
37+
*/
38+
@Description("Reads data from a Reader and splits it into individual records")
39+
@In(Reader.class)
40+
@Out(String.class)
41+
public final class RecordReader extends
42+
DefaultObjectPipe<Reader, ObjectReceiver<String>> {
43+
44+
public static final char DEFAULT_SEPARATOR = '\u001d';
45+
46+
private static final int BUFFER_SIZE = 1024 * 1024 * 16;
47+
48+
private final StringBuilder builder = new StringBuilder();
49+
private final char[] buffer = new char[BUFFER_SIZE];
50+
51+
private char separator = DEFAULT_SEPARATOR;
52+
private boolean skipEmptyRecords = true;
53+
54+
public void setSeparator(final String separator) {
55+
if (separator.length() >= 1) {
56+
this.separator = separator.charAt(0);
57+
} else {
58+
this.separator = DEFAULT_SEPARATOR;
59+
}
60+
}
61+
62+
public void setSeparator(final char separator) {
63+
this.separator = separator;
64+
}
65+
66+
public char getSeparator() {
67+
return separator;
68+
}
69+
70+
public void setSkipEmptyRecords(final boolean skipEmptyRecords) {
71+
this.skipEmptyRecords = skipEmptyRecords;
72+
}
73+
74+
public boolean getSkipEmptyRecords() {
75+
return skipEmptyRecords;
76+
}
77+
78+
@Override
79+
public void process(final Reader reader) {
80+
assert !isClosed();
81+
82+
try {
83+
boolean nothingRead = true;
84+
int size;
85+
while ((size = reader.read(buffer)) != -1) {
86+
nothingRead = false;
87+
int offset = 0;
88+
for (int i = 0; i < size; ++i) {
89+
if (buffer[i] == separator) {
90+
builder.append(buffer, offset, i - offset);
91+
offset = i + 1;
92+
emitRecord();
93+
}
94+
}
95+
builder.append(buffer, offset, size - offset);
96+
}
97+
if (!nothingRead) {
98+
emitRecord();
99+
}
100+
101+
} catch (final IOException e) {
102+
throw new MetafactureException(e);
103+
}
104+
}
105+
106+
private void emitRecord() {
107+
final String record = builder.toString();
108+
if (!skipEmptyRecords || !record.isEmpty()) {
109+
getReceiver().process(record);
110+
builder.delete(0, builder.length());
111+
}
112+
}
113+
114+
}

‎src/main/java/org/culturegraph/mf/stream/pipe/RdfMacroPipe.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
*/
1616
package org.culturegraph.mf.stream.pipe;
1717

18+
import org.apache.commons.lang.StringUtils;
1819
import org.culturegraph.mf.framework.DefaultStreamPipe;
1920
import org.culturegraph.mf.framework.StreamReceiver;
2021
import org.culturegraph.mf.framework.annotations.Description;
@@ -76,7 +77,7 @@ public void endEntity() {
7677
@Override
7778
public void literal(final String name, final String value) {
7879
final int index = name.indexOf(LANGUAGE_MARKER);
79-
if (name.charAt(0)==REFERENCE_MARKER) {
80+
if (StringUtils.isNotEmpty(name) && name.charAt(0)==REFERENCE_MARKER) {
8081
getReceiver().startEntity(name.substring(1));
8182
getReceiver().literal(RDF_REFERENCE, value);
8283
getReceiver().endEntity();

‎src/main/java/org/culturegraph/mf/stream/sink/SimpleXmlWriter.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
*/
1616
package org.culturegraph.mf.stream.sink;
1717

18+
import java.net.URL;
1819
import java.util.ArrayList;
1920
import java.util.Collections;
2021
import java.util.HashMap;
@@ -89,6 +90,13 @@ public void setNamespaceFile(final String file) {
8990
namespaces.put(entry.getKey().toString(), entry.getValue().toString());
9091
}
9192
}
93+
94+
public void setNamespaceFile(final URL url) {
95+
final Properties properties = ResourceUtil.loadProperties(url);
96+
for (final Entry<Object, Object> entry : properties.entrySet()) {
97+
namespaces.put(entry.getKey().toString(), entry.getValue().toString());
98+
}
99+
}
92100

93101
public void setWriteXmlHeader(final boolean writeXmlHeader) {
94102
this.writeXmlHeader = writeXmlHeader;

0 commit comments

Comments
 (0)