Skip to content

Commit 3986e5c

Browse files
dr0idr0i
committed
WIP SRUopener
1 parent b9eebb4 commit 3986e5c

File tree

2 files changed

+258
-0
lines changed

2 files changed

+258
-0
lines changed

‎metafacture-biblio/src/main/java/org/metafacture/biblio/SruOpener.java

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
/* Copyright 2013 Pascal Christoph.
2+
* Licensed under the Eclipse Public License 1.0 */
3+
4+
package org.metafacture.biblio;
5+
6+
import org.metafacture.framework.FluxCommand;
7+
import org.metafacture.framework.MetafactureException;
8+
import org.metafacture.framework.ObjectReceiver;
9+
import org.metafacture.framework.annotations.Description;
10+
import org.metafacture.framework.annotations.In;
11+
import org.metafacture.framework.annotations.Out;
12+
import org.metafacture.framework.helpers.DefaultObjectPipe;
13+
14+
import java.io.IOException;
15+
import java.io.InputStream;
16+
import java.io.InputStreamReader;
17+
import java.io.Reader;
18+
import java.net.HttpURLConnection;
19+
import java.net.URL;
20+
21+
/**
22+
* Opens an SRU (Search Retrieval by URL) stream and passes a reader to the receiver.
23+
* The input should be the base URL of the SRU service to be retrieved from.
24+
*
25+
* @author Pascal Christoph (dr0i)
26+
*/
27+
@Description("Opens a SRU stream and passes a reader to the receiver. The input should be the base URL of the SRU service to be retrieved from. Mandatory argument is: QUERY.")
28+
@In(String.class)
29+
@Out(Reader.class)
30+
@FluxCommand("open-sru")
31+
public final class SruOpener extends DefaultObjectPipe<String, ObjectReceiver<Reader>> {
32+
33+
private static final String OPERATION = "searchRetrieve";
34+
private static final String RECORD_SCHEMA = "MARC21-xml";
35+
private static final String USER_AGENT = "";
36+
private static final String VERSION = "2.0";
37+
38+
private static final int CONNECTION_TIMEOUT = 11000;
39+
private static final int MAXIMUM_RECORDS = 10;
40+
private static final int START_RECORD = 1;
41+
42+
private String operation = OPERATION;
43+
private String query;
44+
private String recordSchema = RECORD_SCHEMA;
45+
private String userAgent = USER_AGENT;
46+
private String version = VERSION;
47+
48+
private int maximumRecords = MAXIMUM_RECORDS;
49+
private int startRecord = START_RECORD;
50+
private int totalRecords;
51+
52+
private boolean stopRetrieving;
53+
54+
55+
/**
56+
* Creates an instance of {@link SruOpener}
57+
*/
58+
public SruOpener() {
59+
}
60+
61+
/**
62+
* Sets the User Agent to use. <strong>Default value: {@value USER_AGENT}</strong>.
63+
*
64+
* @param userAgent a user agent to be used when opening a URL
65+
*/
66+
public void setUserAgent(final String userAgent) {
67+
this.userAgent = userAgent;
68+
}
69+
70+
/**
71+
* Sets the query of the search.
72+
* <strong>Setting a query is mandatory.</strong>
73+
*
74+
* @param query the query
75+
*/
76+
77+
public void setQuery(final String query) {
78+
this.query = query;
79+
}
80+
81+
/**
82+
* Sets total number of records to be retrieved. <strong>Default value: indefinite (as in "all")</strong>.
83+
*
84+
* @param totalRecords total number of records to be retrieved
85+
*/
86+
public void setTotal(final String totalRecords) {
87+
this.totalRecords = Integer.parseInt(totalRecords);
88+
}
89+
90+
/**
91+
* Sets the maximum of records returned in one lookup. <strong>Default value: {@value MAXIMUM_RECORDS}</strong>.
92+
* The lookup is repeated as long as {@link #maximumRecords} is lesser than {@link #totalRecords}.
93+
*
94+
* @param maximumRecords maximum of records returned in one lookup
95+
*/
96+
public void setMaximumRecords(final String maximumRecords) {
97+
this.maximumRecords = Integer.parseInt(maximumRecords);
98+
}
99+
100+
/**
101+
* Sets where to start when retrieving records. <strong>Default value: {@value START_RECORD}</strong>.
102+
*
103+
* @param startRecord where to start when retrieving records
104+
*/
105+
public void setStartRecord(final String startRecord) {
106+
this.startRecord = Integer.parseInt(startRecord);
107+
}
108+
109+
/**
110+
* Sets the format of the retrieved record data. <strong>Default value: {@value RECORD_SCHEMA}</strong>.
111+
*
112+
* @param recordSchema the format of the data of the records
113+
*/
114+
public void setRecordSchema(final String recordSchema) {
115+
this.recordSchema = recordSchema;
116+
}
117+
118+
/**
119+
* Sets the kind of operation of the lookup. <strong>Default value: {@value OPERATION}</strong>.
120+
*
121+
* @param operation the kind of operation of the lookup
122+
*/
123+
public void setOperation(final String operation) {
124+
this.operation = operation;
125+
}
126+
127+
/**
128+
* Sets the version of the lookup. <strong>Default value: {@value VERSION}</strong>.
129+
*
130+
* @param version the version of the lookup
131+
*/
132+
public void setVersion(final String version) {
133+
this.version = version;
134+
}
135+
136+
@Override
137+
public void process(final String baseUrl) {
138+
139+
try {
140+
141+
StringBuilder srUrl = new StringBuilder(baseUrl);
142+
if (query != null) {
143+
srUrl.append("?query=").append(query).append("&operation=").append(operation).append("&recordSchema=").append(recordSchema).append("&version=").append(version);
144+
}
145+
else {
146+
throw new IllegalArgumentException("Missing mandatory parameter 'query'");
147+
}
148+
int retrievedRecords = 0;
149+
while (!stopRetrieving && (totalRecords==0 || retrievedRecords < totalRecords)) {
150+
if (totalRecords >0) {
151+
int yetToRetrieveRecords = retrievedRecords - totalRecords;
152+
if (yetToRetrieveRecords > maximumRecords) {
153+
maximumRecords = yetToRetrieveRecords;
154+
}
155+
}
156+
retrieve(srUrl, startRecord); //todo: bis max lookup zuviel (bis der nämlich sehr klein ist => keine Ergebnisse mehr)
157+
startRecord = startRecord + maximumRecords;
158+
retrievedRecords = retrievedRecords + maximumRecords;
159+
}
160+
}
161+
catch (final IOException e) {
162+
throw new MetafactureException(e);
163+
}
164+
}
165+
166+
private void retrieve(StringBuilder srUrl, int startRecord) throws IOException {
167+
final URL urlToOpen = new URL(srUrl.toString() + "&maximumRecords=" + maximumRecords+"&startRecord=" + startRecord);
168+
final HttpURLConnection connection = (HttpURLConnection) urlToOpen.openConnection();
169+
170+
connection.setConnectTimeout(CONNECTION_TIMEOUT);
171+
if (!userAgent.isEmpty()) {
172+
connection.setRequestProperty("User-Agent", userAgent);
173+
}
174+
InputStream istream = getInputStream(connection);
175+
try (
176+
InputStreamReader inputStreamReader = new InputStreamReader(istream);
177+
) {
178+
System.out.println("srUrl="+srUrl);
179+
System.out.println("startRecord="+startRecord);
180+
System.out.println("istream.length="+istream.available());
181+
if (istream.available() < 768){ // we take it that this is a result without a record
182+
stopRetrieving = true;
183+
}
184+
185+
getReceiver().process(inputStreamReader);
186+
}
187+
}
188+
189+
private InputStream getInputStream(final HttpURLConnection connection) {
190+
try {
191+
return connection.getInputStream();
192+
}
193+
catch (final IOException e) {
194+
stopRetrieving = true;
195+
return connection.getErrorStream();
196+
}
197+
}
198+
199+
}

‎metafacture-biblio/src/test/java/org/metafacture/biblio/SruOpenerTest.java

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package org.metafacture.biblio;
2+
3+
import org.junit.Test;
4+
import org.metafacture.framework.ObjectReceiver;
5+
6+
import java.io.BufferedReader;
7+
import java.io.IOException;
8+
import java.io.Reader;
9+
10+
public class SruOpenerTest {
11+
12+
private StringBuilder resultCollector = new StringBuilder();
13+
private int resultCollectorsResetStreamCount;
14+
15+
16+
@Test
17+
public void test(){
18+
SruOpener sruOpener = new SruOpener();
19+
sruOpener.setReceiver(new ObjectReceiver<Reader> () {
20+
21+
@Override
22+
public void process(final Reader obj) {
23+
BufferedReader in = new BufferedReader(obj);
24+
String line = null;
25+
StringBuilder rslt = new StringBuilder();
26+
while (true) {
27+
try {
28+
if (!((line = in.readLine()) != null)) break;
29+
}
30+
catch (IOException e) {
31+
throw new RuntimeException(e);
32+
}
33+
rslt.append(line);
34+
}
35+
System.out.println(rslt.toString());
36+
resultCollector.append(obj);
37+
}
38+
39+
@Override
40+
public void resetStream() {
41+
++resultCollectorsResetStreamCount;
42+
}
43+
44+
@Override
45+
public void closeStream() {
46+
47+
}
48+
});
49+
50+
// sruOpener.setQuery("dnb.isil%3DDE-Sol1");
51+
sruOpener.setQuery("WVN%3D24A05");
52+
sruOpener.setRecordSchema("MARC21plus-xml");
53+
sruOpener.setVersion("1.1");
54+
sruOpener.setStartRecord("1890");
55+
sruOpener.process("https://services.dnb.de/sru/dnb");
56+
System.out.println(resultCollector.toString());
57+
System.out.println(resultCollector.toString());
58+
}
59+
}

0 commit comments

Comments
 (0)