diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 0eeaa4e6dc8..43eac38df6b 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -22,6 +22,7 @@ env: SpringerNatureAPIKey: ${{ secrets.SpringerNatureAPIKey }} AstrophysicsDataSystemAPIKey: ${{ secrets.AstrophysicsDataSystemAPIKey }} IEEEAPIKey: ${{ secrets.IEEEAPIKey }} + WorldCatAPIKey: ${{ secrets.WorldCatAPIKey }} OSXCERT: ${{ secrets.OSX_SIGNING_CERT }} GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.vfs.watch=false diff --git a/CHANGELOG.md b/CHANGELOG.md index 062b67edfd4..e9887a4a417 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve - We added some symbols and keybindings to the context menu in the entry editor. [#7268](https://github.com/JabRef/jabref/pull/7268) - We added keybindings for setting and clearing the read status. [#7264](https://github.com/JabRef/jabref/issues/7264) - We added two new fields to track the creation and most recent modification date and time for each entry. [koppor#130](https://github.com/koppor/jabref/issues/130) +- We added a fetcher for [WorldCat](https://www.worldcat.org/). [#1065](https://github.com/JabRef/jabref/issues/1065) [#2581](https://github.com/JabRef/jabref/issues/2581) - We added a feature that allows the user to copy highlighted text in the preview window. [#6962](https://github.com/JabRef/jabref/issues/6962) - We added a feature that allows you to create new BibEntry via paste arxivId [#2292](https://github.com/JabRef/jabref/issues/2292) - We added support for conducting automated and systematic literature search across libraries and git support for persistence [#369](https://github.com/koppor/jabref/issues/369) diff --git a/build.gradle b/build.gradle index bd4c4d61ef7..6bee3a9706c 100644 --- a/build.gradle +++ b/build.gradle @@ -294,6 +294,7 @@ processResources { "springerNatureAPIKey": System.getenv('SpringerNatureAPIKey') ? System.getenv('SpringerNatureAPIKey') : '', "astrophysicsDataSystemAPIKey": System.getenv('AstrophysicsDataSystemAPIKey') ? System.getenv('AstrophysicsDataSystemAPIKey') : '', "ieeeAPIKey": System.getenv('IEEEAPIKey') ? System.getenv('IEEEAPIKey') : '', + "worldCatAPIKey": System.getenv('worldCatAPIKey') ? System.getenv('worldCatAPIKey') : '', "scienceDirectApiKey": System.getenv('SCIENCEDIRECTAPIKEY') ? System.getenv('SCIENCEDIRECTAPIKEY') : '' ) filteringCharset = 'UTF-8' diff --git a/docs/advanced-reading/fetchers.md b/docs/advanced-reading/fetchers.md index a28445d66b9..12776591794 100644 --- a/docs/advanced-reading/fetchers.md +++ b/docs/advanced-reading/fetchers.md @@ -9,9 +9,10 @@ Fetchers are the implementation of the [search using online services](https://do | [SAO/NASA Astrophysics Data System](https://docs.jabref.org/collect/import-using-online-bibliographic-database#sao-nasa-astrophysics-data-system) | [ADS UI](https://ui.adsabs.harvard.edu/user/settings/token) | `AstrophysicsDataSystemAPIKey` | 5000 calls/day | | [ScienceDirect](https://www.sciencedirect.com/) | | `ScienceDirectApiKey` | | | [Springer Nature](https://docs.jabref.org/collect/import-using-online-bibliographic-database#springer) | [Springer Nature API Portal](https://dev.springernature.com/) | `SpringerNatureAPIKey` | 5000 calls/day | +| [WorldCat](https://www.worldcat.org/) | [OCLC Developer Network](https://www.oclc.org/developer/develop/authentication/how-to-request-a-wskey.en.html) | `worldCatAPIKey` | \(none\) | | [Zentralblatt Math](https://www.zbmath.org/) | \(none\) | \(none\) | Depending on the current network | -"Depending on the current network" means that it depends whether your request is routed through a network having paid access. For instance, some universities have subscriptions to MathSciNet. +"Depending on the current network" means that it depends on whether your request is routed through a network having paid access. For instance, some universities have subscriptions to MathSciNet. On Windows, you have to log-off and log-on to let IntelliJ know about the environment variable change. Execute the gradle task "processResources" in the group "others" within IntelliJ to ensure the values have been correctly written. Now, the fetcher tests should run without issues. diff --git a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java index 7665da90056..4a7cabadd62 100644 --- a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java +++ b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java @@ -352,7 +352,7 @@ private void setupToolBar() { // Add menu for fetching bibliographic information ContextMenu fetcherMenu = new ContextMenu(); - for (EntryBasedFetcher fetcher : WebFetchers.getEntryBasedFetchers(preferencesService.getImportFormatPreferences())) { + for (EntryBasedFetcher fetcher : WebFetchers.getEntryBasedFetchers(preferencesService.getImportFormatPreferences(), preferencesService.getApiKeyPreferences())) { MenuItem fetcherMenuItem = new MenuItem(fetcher.getName()); fetcherMenuItem.setOnAction(event -> fetchAndMerge(fetcher)); fetcherMenu.getItems().add(fetcherMenuItem); diff --git a/src/main/java/org/jabref/gui/preferences/PreferencesDialogViewModel.java b/src/main/java/org/jabref/gui/preferences/PreferencesDialogViewModel.java index e2e378c3811..b469ddd29db 100644 --- a/src/main/java/org/jabref/gui/preferences/PreferencesDialogViewModel.java +++ b/src/main/java/org/jabref/gui/preferences/PreferencesDialogViewModel.java @@ -19,6 +19,7 @@ import org.jabref.gui.preferences.entryeditortabs.CustomEditorFieldsTab; import org.jabref.gui.preferences.exporter.ExportCustomizationTab; import org.jabref.gui.preferences.external.ExternalTab; +import org.jabref.gui.preferences.fetcher.FetcherTab; import org.jabref.gui.preferences.file.FileTab; import org.jabref.gui.preferences.general.GeneralTab; import org.jabref.gui.preferences.groups.GroupsTab; diff --git a/src/main/java/org/jabref/gui/preferences/fetcher/FetcherTab.fxml b/src/main/java/org/jabref/gui/preferences/fetcher/FetcherTab.fxml new file mode 100644 index 00000000000..c75d6cbc03e --- /dev/null +++ b/src/main/java/org/jabref/gui/preferences/fetcher/FetcherTab.fxml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + diff --git a/src/main/java/org/jabref/gui/preferences/fetcher/FetcherTab.java b/src/main/java/org/jabref/gui/preferences/fetcher/FetcherTab.java new file mode 100644 index 00000000000..97f90faaa24 --- /dev/null +++ b/src/main/java/org/jabref/gui/preferences/fetcher/FetcherTab.java @@ -0,0 +1,41 @@ +package org.jabref.gui.preferences.fetcher; + +import javafx.fxml.FXML; +import javafx.scene.control.CheckBox; +import javafx.scene.control.TextField; + +import org.jabref.gui.preferences.AbstractPreferenceTabView; +import org.jabref.gui.preferences.PreferencesTab; +import org.jabref.logic.l10n.Localization; + +import com.airhacks.afterburner.views.ViewLoader; + +public class FetcherTab extends AbstractPreferenceTabView implements PreferencesTab { + + @FXML private CheckBox useWorldcatKey; + @FXML private TextField worldcatKey; + + public FetcherTab() { + ViewLoader.view(this) + .root(this) + .load(); + } + + @FXML + private void initialize() { + viewModel = new FetcherTabViewModel(preferencesService); + + useWorldcatKey.selectedProperty().bindBidirectional(viewModel.getUseWorldcatKeyProperty()); + worldcatKey.textProperty().bindBidirectional(viewModel.getWorldcatKeyProperty()); + } + + @Override + public String getTabName() { + return Localization.lang("Fetcher API keys"); + } + + @FXML + private void openWorldcatWebpage() { + viewModel.openWorldcatWebpage(); + } +} diff --git a/src/main/java/org/jabref/gui/preferences/fetcher/FetcherTabViewModel.java b/src/main/java/org/jabref/gui/preferences/fetcher/FetcherTabViewModel.java new file mode 100644 index 00000000000..2d3d2e737b2 --- /dev/null +++ b/src/main/java/org/jabref/gui/preferences/fetcher/FetcherTabViewModel.java @@ -0,0 +1,74 @@ +package org.jabref.gui.preferences.fetcher; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javafx.beans.property.BooleanProperty; +import javafx.beans.property.SimpleBooleanProperty; +import javafx.beans.property.SimpleStringProperty; +import javafx.beans.property.StringProperty; + +import org.jabref.gui.desktop.JabRefDesktop; +import org.jabref.gui.preferences.PreferenceTabViewModel; +import org.jabref.logic.importer.FetcherApiPreferences; +import org.jabref.model.strings.StringUtil; +import org.jabref.preferences.PreferencesService; + +public class FetcherTabViewModel implements PreferenceTabViewModel { + + private final String WORLDCAT_REQUEST_URL = "https://platform.worldcat.org/wskey"; + + private final BooleanProperty useWorldcatKeyProperty = new SimpleBooleanProperty(); + private final StringProperty worldcatKeyProperty = new SimpleStringProperty(); + + private final PreferencesService preferencesService; + private final FetcherApiPreferences initialFetcherApiPreferences; + + public FetcherTabViewModel(PreferencesService preferencesService) { + this.preferencesService = preferencesService; + this.initialFetcherApiPreferences = preferencesService.getApiKeyPreferences(); + } + + @Override + public void setValues() { + if (StringUtil.isNotBlank(initialFetcherApiPreferences.getWorldcatKey())) { + useWorldcatKeyProperty.setValue(true); + worldcatKeyProperty.setValue(initialFetcherApiPreferences.getWorldcatKey()); + } + } + + @Override + public void storeSettings() { + Map keys = new HashMap<>(); + + keys.put("worldcat", useWorldcatKeyProperty.getValue() ? worldcatKeyProperty.getValue() : ""); + + preferencesService.storeApiKeyPreferences(new FetcherApiPreferences( + keys.get("worldcat")) + ); + } + + public void openWorldcatWebpage() { + JabRefDesktop.openBrowserShowPopup(WORLDCAT_REQUEST_URL); + } + + @Override + public boolean validateSettings() { + return true; + } + + @Override + public List getRestartWarnings() { + return Collections.emptyList(); + } + + public BooleanProperty getUseWorldcatKeyProperty() { + return useWorldcatKeyProperty; + } + + public StringProperty getWorldcatKeyProperty() { + return worldcatKeyProperty; + } +} diff --git a/src/main/java/org/jabref/logic/importer/FetcherApiPreferences.java b/src/main/java/org/jabref/logic/importer/FetcherApiPreferences.java new file mode 100644 index 00000000000..0b022006b3a --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/FetcherApiPreferences.java @@ -0,0 +1,13 @@ +package org.jabref.logic.importer; + +public class FetcherApiPreferences { + private final String worldcatKey; + + public FetcherApiPreferences(String worldcatKey) { + this.worldcatKey = worldcatKey; + } + + public String getWorldcatKey() { + return worldcatKey; + } +} diff --git a/src/main/java/org/jabref/logic/importer/WebFetchers.java b/src/main/java/org/jabref/logic/importer/WebFetchers.java index a30b86db3d0..37b1391036e 100644 --- a/src/main/java/org/jabref/logic/importer/WebFetchers.java +++ b/src/main/java/org/jabref/logic/importer/WebFetchers.java @@ -36,6 +36,7 @@ import org.jabref.logic.importer.fetcher.SpringerFetcher; import org.jabref.logic.importer.fetcher.SpringerLink; import org.jabref.logic.importer.fetcher.TitleFetcher; +import org.jabref.logic.importer.fetcher.WorldcatFetcher; import org.jabref.logic.importer.fetcher.ZbMATH; import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.StandardField; @@ -133,13 +134,14 @@ public static SortedSet getIdBasedFetchers(ImportFormatPreferenc /** * @return sorted set containing entry based fetchers */ - public static SortedSet getEntryBasedFetchers(ImportFormatPreferences importFormatPreferences) { + public static SortedSet getEntryBasedFetchers(ImportFormatPreferences importFormatPreferences, FetcherApiPreferences fetcherApiPreferences) { SortedSet set = new TreeSet<>(Comparator.comparing(WebFetcher::getName)); set.add(new AstrophysicsDataSystem(importFormatPreferences)); set.add(new DoiFetcher(importFormatPreferences)); set.add(new IsbnFetcher(importFormatPreferences)); set.add(new MathSciNet(importFormatPreferences)); set.add(new CrossRef()); + set.add(new WorldcatFetcher(fetcherApiPreferences.getWorldcatKey())); set.add(new ZbMATH(importFormatPreferences)); return set; } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java new file mode 100644 index 00000000000..cc29203000c --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/WorldcatFetcher.java @@ -0,0 +1,217 @@ +package org.jabref.logic.importer.fetcher; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import org.jabref.logic.importer.EntryBasedFetcher; +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.ParserResult; +import org.jabref.logic.importer.fileformat.WorldcatImporter; +import org.jabref.logic.net.URLDownload; +import org.jabref.logic.util.BuildInfo; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.strings.StringUtil; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +/** + * EntryBasedFetcher that searches the Worldcat database + * + * @see Worldcat API documentation + */ +public class WorldcatFetcher implements EntryBasedFetcher { + + private String WORLDCAT_OPEN_SEARCH_URL = "https://www.worldcat.org/webservices/catalog/search/opensearch?wskey="; + private String WORLDCAT_READ_URL = "https://www.worldcat.org/webservices/catalog/content/{OCLC-NUMBER}?recordSchema=info%3Asrw%2Fschema%2F1%2Fdc&wskey="; + + public WorldcatFetcher(String worldcatKey) { + if (StringUtil.isBlank(worldcatKey)) { + worldcatKey = new BuildInfo().worldCatAPIKey; + } + + WORLDCAT_OPEN_SEARCH_URL += worldcatKey; + WORLDCAT_READ_URL += worldcatKey; + } + + @Override + public String getName() { + return "Worldcat Fetcher"; + } + + /** + * Create a open search query with specified title + * @param title the title to include in the query + * @return the earch query for the api + */ + private String getOpenSearchURL(String title) throws MalformedURLException { + String query = "&q=srw.ti+all+" + URLEncoder.encode("\"" + title + "\"", StandardCharsets.UTF_8); + URL url = new URL(WORLDCAT_OPEN_SEARCH_URL + query); + return url.toString(); + } + + /** + * Make request to open search API of Worldcat, with specified title + * @param title the title of the search + * @return the body of the HTTP response + */ + private String makeOpenSearchRequest(String title) throws FetcherException { + try { + URLDownload urlDownload = new URLDownload(getOpenSearchURL(title)); + URLDownload.bypassSSLVerification(); + return urlDownload.asString(); + } catch (MalformedURLException e) { + throw new FetcherException("Bad url", e); + } catch (IOException e) { + throw new FetcherException("Error with Open Search Request (Worldcat)", e); + } + } + + /** + * Get more information about a article through its OCLC id. Picks the first + * element with this tag + * @param id the oclc id + * @return the Node of the XML element that contains all tags + */ + private Node getSpecificInfoOnOCLC(String id) throws IOException { + URLDownload urlDownload = new URLDownload(WORLDCAT_READ_URL.replace("{OCLC-NUMBER}", id)); + URLDownload.bypassSSLVerification(); + String resp = urlDownload.asString(); + + Document mainDoc = parse(resp); + + return mainDoc.getElementsByTagName("oclcdcs").item(0); + } + + /** + * Parse a string to an xml document + * @param s the string to be parsed + * @return XML document representing the content of s + * @throws IllegalArgumentException if s is badly formated or other exception occurs during parsing + */ + private Document parse(String s) { + try (BufferedReader r = new BufferedReader(new StringReader(s))) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + + return builder.parse(new InputSource(r)); + } catch (ParserConfigurationException e) { + throw new IllegalArgumentException("Parser Config Exception: " + e.getMessage(), e); + } catch (SAXException e) { + throw new IllegalArgumentException("SAX Exception: " + e.getMessage(), e); + } catch (IOException e) { + throw new IllegalArgumentException("IO Exception: " + e.getMessage(), e); + } + } + + /** + * Parse OpenSearch XML to retrieve OCLC-ids and fetch details about those. + * Returns new XML doc in the form + *
+     * {@literal }
+     *      {@literal }
+     *          OCLC DETAIL
+     *      {@literal }
+     *      ...
+     * {@literal }
+     * 
+ * @param doc the open search result + * @return the new xml document + * @throws FetcherException if the fetcher fails to parse the result + */ + private Document parseOpenSearchXML(Document doc) throws FetcherException { + try { + Element feed = (Element) doc.getElementsByTagName("feed").item(0); + NodeList entryXMLList = feed.getElementsByTagName("entry"); + + DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + + Document newDoc = docBuilder.newDocument(); + + Element root = newDoc.createElement("entries"); + newDoc.appendChild(root); + for (int i = 0; i < entryXMLList.getLength(); i++) { + Element xmlEntry = (Element) entryXMLList.item(i); + + String oclc = xmlEntry.getElementsByTagName("oclcterms:recordIdentifier").item(0).getTextContent(); + Element detailedInfo = (Element) newDoc.importNode(getSpecificInfoOnOCLC(oclc), true); + + Element newEntry = newDoc.createElement("entry"); + newEntry.appendChild(detailedInfo); + + root.appendChild(newEntry); + } + return newDoc; + } catch (ParserConfigurationException e) { + throw new FetcherException("Error with XML creation (Worldcat fetcher)", e); + } catch (IOException e) { + throw new FetcherException("Error with OCLC parsing (Worldcat fetcher)", e); + } + } + + @Override + public List performSearch(BibEntry entry) throws FetcherException { + Optional entryTitle = entry.getLatexFreeField(StandardField.TITLE); + if (entryTitle.isPresent()) { + String openSearchXMLResponse = makeOpenSearchRequest(entryTitle.get()); + Document openSearchDocument = parse(openSearchXMLResponse); + + Document detailedXML = parseOpenSearchXML(openSearchDocument); + String detailedXMLString; + try (StringWriter sw = new StringWriter()) { + // Transform XML to String + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer t = tf.newTransformer(); + t.transform(new DOMSource(detailedXML), new StreamResult(sw)); + detailedXMLString = sw.toString(); + } catch (TransformerException e) { + throw new FetcherException("Could not transform XML", e); + } catch (IOException e) { + throw new FetcherException("Could not close StringWriter", e); + } + + WorldcatImporter importer = new WorldcatImporter(); + ParserResult parserResult; + try { + if (importer.isRecognizedFormat(detailedXMLString)) { + parserResult = importer.importDatabase(detailedXMLString); + } else { + // For displaying An ErrorMessage + BibDatabase errorBibDataBase = new BibDatabase(); + parserResult = new ParserResult(errorBibDataBase); + } + return parserResult.getDatabase().getEntries(); + } catch (IOException e) { + throw new FetcherException("Could not perform search (Worldcat) ", e); + } + } else { + return new ArrayList<>(); + } + } + +} diff --git a/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java b/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java new file mode 100644 index 00000000000..d324d1cf484 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fileformat/WorldcatImporter.java @@ -0,0 +1,138 @@ +package org.jabref.logic.importer.fileformat; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.jabref.logic.importer.Importer; +import org.jabref.logic.importer.ParserResult; +import org.jabref.logic.util.FileType; +import org.jabref.logic.util.StandardFileType; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +public class WorldcatImporter extends Importer { + + private static final String DESCRIPTION = "Importer for Worldcat Open Search XML format"; + private static final String NAME = "WorldcatImporter"; + + /** + * Parse the reader to an xml document + * + * @param s the reader to be parsed + * @return XML document representing the content of s + * @throws IllegalArgumentException if s is badly formated or other exception occurs during parsing + */ + private Document parse(BufferedReader s) { + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + + return builder.parse(new InputSource(s)); + } catch (ParserConfigurationException e) { + throw new IllegalArgumentException("Parser Config Exception: ", e); + } catch (SAXException e) { + throw new IllegalArgumentException("SAX Exception: ", e); + } catch (IOException e) { + throw new IllegalArgumentException("IO Exception: ", e); + } + } + + @Override + public boolean isRecognizedFormat(BufferedReader input) throws IOException { + try { + Document doc = parse(input); + return doc.getElementsByTagName("feed") != null; + } catch (IllegalArgumentException e) { + return false; + } + } + + /** + * Gets the text content of the given element. If the element was not found, an empty Optional is returned; + * + * @param xml the element do search trough + * @param tag the tag to find + */ + private Optional getElementContent(Element xml, String tag) { + NodeList nl = xml.getElementsByTagName(tag); + if (nl == null) { + return Optional.empty(); + } + if (nl.getLength() == 0) { + return Optional.empty(); + } + return Optional.ofNullable(nl.item(0).getTextContent()); + } + + /** + * Parse the xml entry to a bib entry + * + * @param xmlEntry the XML element from open search + * @return the correspoinding bibentry + */ + private BibEntry xmlEntryToBibEntry(Element xmlEntry) { + BibEntry result = new BibEntry(); + + getElementContent(xmlEntry, "dc:creator").ifPresent(content -> result.setField(StandardField.AUTHOR, content)); + getElementContent(xmlEntry, "dc:title").ifPresent(content -> result.setField(StandardField.TITLE, content)); + getElementContent(xmlEntry, "dc:date").ifPresent(content -> result.setField(StandardField.YEAR, content)); + getElementContent(xmlEntry, "dc:publisher").ifPresent(content -> result.setField(StandardField.PUBLISHER, content)); + getElementContent(xmlEntry, "dc:recordIdentifier").ifPresent(content -> result.setField(StandardField.URL, "http://worldcat.org/oclc/" + content)); + + return result; + } + + /** + * Parse an XML documents with open search entries to a parserResult of the bibentries + * + * @param doc the main XML document from open search + * @return the ParserResult containing the BibEntries collection + */ + private ParserResult docToParserRes(Document doc) { + Element feed = (Element) doc.getElementsByTagName("entries").item(0); + NodeList entryXMLList = feed.getElementsByTagName("entry"); + + List bibList = new ArrayList<>(entryXMLList.getLength()); + for (int i = 0; i < entryXMLList.getLength(); i++) { + Element xmlEntry = (Element) entryXMLList.item(i); + BibEntry bibEntry = xmlEntryToBibEntry(xmlEntry); + bibList.add(bibEntry); + } + + return new ParserResult(bibList); + } + + @Override + public ParserResult importDatabase(BufferedReader input) throws IOException { + Document parsedDoc = parse(input); + return docToParserRes(parsedDoc); + } + + @Override + public String getName() { + return NAME; + } + + @Override + public FileType getFileType() { + return StandardFileType.XML; + } + + @Override + public String getDescription() { + return DESCRIPTION; + } +} diff --git a/src/main/java/org/jabref/logic/util/BuildInfo.java b/src/main/java/org/jabref/logic/util/BuildInfo.java index 47ef962f21a..998258f161c 100644 --- a/src/main/java/org/jabref/logic/util/BuildInfo.java +++ b/src/main/java/org/jabref/logic/util/BuildInfo.java @@ -27,6 +27,7 @@ public final class BuildInfo { public final String astrophysicsDataSystemAPIKey; public final String ieeeAPIKey; public final String scienceDirectApiKey; + public final String worldCatAPIKey; public final String minRequiredJavaVersion; public final boolean allowJava9; @@ -55,6 +56,7 @@ public BuildInfo(String path) { astrophysicsDataSystemAPIKey = BuildInfo.getValue(properties, "astrophysicsDataSystemAPIKey", "tAhPRKADc6cC26mZUnAoBt3MAjCvKbuCZsB4lI3c"); ieeeAPIKey = BuildInfo.getValue(properties, "ieeeAPIKey", "5jv3wyt4tt2bwcwv7jjk7pc3"); scienceDirectApiKey = BuildInfo.getValue(properties, "scienceDirectApiKey", "fb82f2e692b3c72dafe5f4f1fa0ac00b"); + worldCatAPIKey = BuildInfo.getValue(properties, "worldCatAPIKey", ""); minRequiredJavaVersion = properties.getProperty("minRequiredJavaVersion", "1.8"); allowJava9 = "true".equals(properties.getProperty("allowJava9", "true")); } diff --git a/src/main/java/org/jabref/preferences/JabRefPreferences.java b/src/main/java/org/jabref/preferences/JabRefPreferences.java index ba5df0fa216..d7262d5b44b 100644 --- a/src/main/java/org/jabref/preferences/JabRefPreferences.java +++ b/src/main/java/org/jabref/preferences/JabRefPreferences.java @@ -73,6 +73,7 @@ import org.jabref.logic.cleanup.FieldFormatterCleanups; import org.jabref.logic.exporter.SavePreferences; import org.jabref.logic.exporter.TemplateExporter; +import org.jabref.logic.importer.FetcherApiPreferences; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.fetcher.DoiFetcher; import org.jabref.logic.importer.fileformat.CustomImporter; @@ -272,6 +273,7 @@ public class JabRefPreferences implements PreferencesService { public static final String CLEANUP_FORMATTERS = "CleanUpFormatters"; public static final String IMPORT_FILENAMEPATTERN = "importFileNamePattern"; public static final String IMPORT_FILEDIRPATTERN = "importFileDirPattern"; + public static final String IMPORT_API_WORLDCAT = "importAPIWorldcat"; public static final String NAME_FORMATTER_VALUE = "nameFormatterFormats"; public static final String NAME_FORMATER_KEY = "nameFormatterNames"; public static final String PUSH_TO_APPLICATION = "pushToApplication"; @@ -646,6 +648,8 @@ private JabRefPreferences() { // Download files by default defaults.put(DOWNLOAD_LINKED_FILES, true); + defaults.put(IMPORT_API_WORLDCAT, ""); + String defaultExpression = "**/.*[citationkey].*\\\\.[extension]"; defaults.put(AUTOLINK_REG_EXP_SEARCH_EXPRESSION_KEY, defaultExpression); defaults.put(AUTOLINK_USE_REG_EXP_SEARCH_KEY, Boolean.FALSE); @@ -2341,6 +2345,16 @@ private void purgeCustomExportFormats(int from) { } } + @Override + public FetcherApiPreferences getApiKeyPreferences() { + return new FetcherApiPreferences(get(IMPORT_API_WORLDCAT)); + } + + @Override + public void storeApiKeyPreferences(FetcherApiPreferences preferences) { + put(IMPORT_API_WORLDCAT, preferences.getWorldcatKey()); + } + //************************************************************************************************************* // Preview preferences //************************************************************************************************************* diff --git a/src/main/java/org/jabref/preferences/PreferencesService.java b/src/main/java/org/jabref/preferences/PreferencesService.java index 70fa7bcfd07..e77990454b4 100644 --- a/src/main/java/org/jabref/preferences/PreferencesService.java +++ b/src/main/java/org/jabref/preferences/PreferencesService.java @@ -27,6 +27,7 @@ import org.jabref.logic.cleanup.CleanupPreset; import org.jabref.logic.exporter.SavePreferences; import org.jabref.logic.exporter.TemplateExporter; +import org.jabref.logic.importer.FetcherApiPreferences; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.fileformat.CustomImporter; import org.jabref.logic.importer.importsettings.ImportSettingsPreferences; @@ -303,6 +304,10 @@ public interface PreferencesService { void storeImportSettingsPreferences(ImportSettingsPreferences preferences); + FetcherApiPreferences getApiKeyPreferences(); + + void storeApiKeyPreferences(FetcherApiPreferences preferences); + //************************************************************************************************************* // Preview preferences //************************************************************************************************************* diff --git a/src/main/resources/build.properties b/src/main/resources/build.properties index ec86a4cab54..4a7b1d460fa 100644 --- a/src/main/resources/build.properties +++ b/src/main/resources/build.properties @@ -5,3 +5,4 @@ azureInstrumentationKey=${azureInstrumentationKey} springerNatureAPIKey=${springerNatureAPIKey} astrophysicsDataSystemAPIKey=${astrophysicsDataSystemAPIKey} ieeeAPIKey=${ieeeAPIKey} +worldCatAPIKey=${worldCatAPIKey} diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index c6a9db1ab7c..a86bca4bd53 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -1009,6 +1009,11 @@ No\ search\ matches.=No search matches. The\ output\ option\ depends\ on\ a\ valid\ input\ option.=The output option depends on a valid input option. Linked\ file\ name\ conventions=Linked file name conventions Filename\ format\ pattern=Filename format pattern + +Request\ a\ key\ on\ the\ WorldCat\ platform.=Request a key on the WorldCat platform. +Fetcher\ API\ keys=Fetcher API keys +WorldCat=WorldCat + Additional\ parameters=Additional parameters Cite\ selected\ entries\ between\ parenthesis=Cite selected entries between parenthesis Cite\ selected\ entries\ with\ in-text\ citation=Cite selected entries with in-text citation diff --git a/src/test/java/org/jabref/logic/importer/WebFetchersTest.java b/src/test/java/org/jabref/logic/importer/WebFetchersTest.java index 458a5b0985d..55677479073 100644 --- a/src/test/java/org/jabref/logic/importer/WebFetchersTest.java +++ b/src/test/java/org/jabref/logic/importer/WebFetchersTest.java @@ -26,18 +26,20 @@ class WebFetchersTest { + private FetcherApiPreferences fetcherApiPreferences; private ImportFormatPreferences importFormatPreferences; - private final ClassGraph classGraph = new ClassGraph().enableAllInfo().whitelistPackages("org.jabref"); + private final ClassGraph classGraph = new ClassGraph().enableAllInfo().acceptPackages("org.jabref"); @BeforeEach - void setUp() throws Exception { + void setUp() { importFormatPreferences = mock(ImportFormatPreferences.class); FieldContentFormatterPreferences fieldContentFormatterPreferences = mock(FieldContentFormatterPreferences.class); when(importFormatPreferences.getFieldContentFormatterPreferences()).thenReturn(fieldContentFormatterPreferences); + fetcherApiPreferences = mock(FetcherApiPreferences.class); } @Test - void getIdBasedFetchersReturnsAllFetcherDerivingFromIdBasedFetcher() throws Exception { + void getIdBasedFetchersReturnsAllFetcherDerivingFromIdBasedFetcher() { Set idFetchers = WebFetchers.getIdBasedFetchers(importFormatPreferences); try (ScanResult scanResult = classGraph.scan()) { @@ -61,8 +63,8 @@ void getIdBasedFetchersReturnsAllFetcherDerivingFromIdBasedFetcher() throws Exce } @Test - void getEntryBasedFetchersReturnsAllFetcherDerivingFromEntryBasedFetcher() throws Exception { - Set idFetchers = WebFetchers.getEntryBasedFetchers(importFormatPreferences); + void getEntryBasedFetchersReturnsAllFetcherDerivingFromEntryBasedFetcher() { + Set idFetchers = WebFetchers.getEntryBasedFetchers(importFormatPreferences, fetcherApiPreferences); try (ScanResult scanResult = classGraph.scan()) { ClassInfoList controlClasses = scanResult.getClassesImplementing(EntryBasedFetcher.class.getCanonicalName()); @@ -75,7 +77,7 @@ void getEntryBasedFetchersReturnsAllFetcherDerivingFromEntryBasedFetcher() throw } @Test - void getSearchBasedFetchersReturnsAllFetcherDerivingFromSearchBasedFetcher() throws Exception { + void getSearchBasedFetchersReturnsAllFetcherDerivingFromSearchBasedFetcher() { Set searchBasedFetchers = WebFetchers.getSearchBasedFetchers(importFormatPreferences); try (ScanResult scanResult = classGraph.scan()) { ClassInfoList controlClasses = scanResult.getClassesImplementing(SearchBasedFetcher.class.getCanonicalName()); @@ -99,7 +101,7 @@ void getSearchBasedFetchersReturnsAllFetcherDerivingFromSearchBasedFetcher() thr } @Test - void getFullTextFetchersReturnsAllFetcherDerivingFromFullTextFetcher() throws Exception { + void getFullTextFetchersReturnsAllFetcherDerivingFromFullTextFetcher() { Set fullTextFetchers = WebFetchers.getFullTextFetchers(importFormatPreferences); try (ScanResult scanResult = classGraph.scan()) { @@ -115,7 +117,7 @@ void getFullTextFetchersReturnsAllFetcherDerivingFromFullTextFetcher() throws Ex } @Test - void getIdFetchersReturnsAllFetcherDerivingFromIdFetcher() throws Exception { + void getIdFetchersReturnsAllFetcherDerivingFromIdFetcher() { Set> idFetchers = WebFetchers.getIdFetchers(importFormatPreferences); try (ScanResult scanResult = classGraph.scan()) { diff --git a/src/test/java/org/jabref/logic/importer/fetcher/WorldcatFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/WorldcatFetcherTest.java new file mode 100644 index 00000000000..345495dbaaa --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/WorldcatFetcherTest.java @@ -0,0 +1,42 @@ +package org.jabref.logic.importer.fetcher; + +import java.util.Collections; +import java.util.List; + +import org.jabref.logic.importer.FetcherException; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.testutils.category.FetcherTest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +@FetcherTest +public class WorldcatFetcherTest { + + private WorldcatFetcher fetcher; + + @BeforeEach + public void setUp() { + fetcher = new WorldcatFetcher("aMHOf2rfzUt3fuKkb7DXX8pkBv1AmcBWwwoSfwpt8CMhdUdxXscB4ESOmBPs4NlmYJmFtcSZ3Q5kMxzb"); + } + + @Test + public void testPerformSearchForBadTitle() throws FetcherException { + BibEntry entry = new BibEntry(); + // Mashing keyboard. Verified on https://platform.worldcat.org/api-explorer/apis/wcapi/Bib/OpenSearch + entry.setField(StandardField.TITLE, "ASDhbsdfnm"); + List list = fetcher.performSearch(entry); + assertEquals(Collections.emptyList(), list); + } + + @Test + public void testPerformSearchForExistingTitle() throws FetcherException { + BibEntry entry = new BibEntry().withField(StandardField.TITLE, "Markdown architectural decision records: Format and tool support"); + List list = fetcher.performSearch(entry); + assertFalse(list.isEmpty()); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fileformat/WorldcatImporterTest.java b/src/test/java/org/jabref/logic/importer/fileformat/WorldcatImporterTest.java new file mode 100644 index 00000000000..a570853d17e --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fileformat/WorldcatImporterTest.java @@ -0,0 +1,84 @@ +package org.jabref.logic.importer.fileformat; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.function.Predicate; + +import org.jabref.logic.importer.ParserResult; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class WorldcatImporterTest { + + WorldcatImporter importer; + + private String getFilePath(String filename) throws IOException { + Predicate filePredicate = name -> name.startsWith(filename) && name.endsWith(".xml"); + Collection paths = ImporterTestEngine.getTestFiles(filePredicate); + if (paths.size() != 1) { + throw new IllegalArgumentException("Filename returned 0 or more than 1 result: " + filename); + } + return paths.iterator().next(); + } + + private String getFileContent(String filename) throws IOException { + String path = getFilePath(filename); + return Files.readString(getPath(path)); + } + + private static Path getPath(String fileName) throws IOException { + try { + return Path.of(ImporterTestEngine.class.getResource(fileName).toURI()); + } catch (URISyntaxException e) { + throw new IOException(e); + } + } + + @BeforeEach + public void setUp() { + importer = new WorldcatImporter(); + + } + + @Test + public void withResultIsRecognizedFormat() throws IOException { + ImporterTestEngine.testIsRecognizedFormat(importer, getFilePath("WorldcatImporterTestWithResult")); + } + + @Test + public void withoutResultIsRecognizedFormat() throws IOException { + ImporterTestEngine.testIsRecognizedFormat(importer, getFilePath("WorldcatImporterTestWithoutResult")); + } + + @Test + public void badXMLIsNotRecognizedFormat() throws IOException { + boolean isReq = importer.isRecognizedFormat("Nah bruh"); + assertFalse(isReq); + } + + @Disabled("Will not work without API key") + @Test + public void withResultReturnsNonEmptyResult() throws IOException { + String withResultXML = getFileContent("WorldcatImporterTestWithResult"); + ParserResult res = importer.importDatabase(withResultXML); + assertTrue(res.getDatabase().getEntries().size() > 0); + } + + @Disabled("Will not work without API key") + @Test + public void withoutResultReturnsEmptyResult() throws IOException { + String withoutResultXML = getFileContent("WorldcatImporterTestWithResult"); + ParserResult res = importer.importDatabase(withoutResultXML); + assertEquals(0, res.getDatabase().getEntries().size()); + } + +} diff --git a/src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithResult.xml b/src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithResult.xml new file mode 100644 index 00000000000..7cac36c96ac --- /dev/null +++ b/src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithResult.xml @@ -0,0 +1,38 @@ + + + 1994 + Reading between the texts : Benjamin Thomas's Abraham Lincoln and Stephen Oates's With malice towards none / Robert Bray--"A horse chestnut is not a chestnut horse" : a refutation of Bray, Davis, MacGregor, and Wollan--Stephen B. Oates. + 2 volumes ; 23 cm. + eng + McFarland + Bray, Robert. Reading between the texts : Benjamin Thomas's Abraham Lincoln and Stephen Oates's With malice toward none. + Oates, Stephen B. Horse chestnut is not a chestnut horse : a refutation of Bray, Davis, MacGregor, and Wollan. + Journal of information ethics, v. 3 no. 1-2 + Oates, Stephen B. With malice toward none. + Thomas, Benjamin Platt, 1902-1956. Abraham Lincoln. + Plagiarism--United States. + Plagiarism. + Text + 950303 + 32079910 + + + Virgin records international. + Virgin France. + Virgin Schallplatten Gmbh. + Sparks. + 1997 (P) + Texte des chansons. + Munchen 40 : prod. Virgin Schallplatten GmbH, P 1997. + Notice en anglais / Keaton-Welles, Amelia. + 1 disque compact + 1 brochure ([20] p.) : ill. en coul. ; 12 cm + eng + Virgin records international + Distrib. Virgin France + http://catalogue.bnf.fr/ark:/12148/cb383781119 + Plagiarism + Sound + 971008 + 659047837 + + diff --git a/src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithoutResult.xml b/src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithoutResult.xml new file mode 100644 index 00000000000..0dc66c5963d --- /dev/null +++ b/src/test/resources/org/jabref/logic/importer/fileformat/WorldcatImporterTestWithoutResult.xml @@ -0,0 +1,2 @@ + +