Refchecker Logic Functionality

jablib/src/main/java/org/jabref/logic/database/DuplicateCheck.java

@@ -28,6 +28,7 @@
 import org.jabref.model.strings.StringUtil;
 
 import com.google.common.collect.Sets;
+import org.apache.commons.text.similarity.LevenshteinDistance;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -364,4 +365,34 @@ public Optional<BibEntry> containsDuplicate(final BibDatabase database,
 
         return database.getEntries().stream().filter(other -> isDuplicate(entry, other, bibDatabaseMode)).findFirst();
     }
+
+    /**
+     * Checks across all fields of the entries,
+     * any matching ones get compared.
+     * If they are not the same the score goes down.
+     * The score goes down depending on the
+     * Levenshtein distance between the two entries.
+     * <p>
+     * If the result is zero, it means that either no common fields were found
+     * or that all common fields were very far apart lexically.
+     * <p>
+     * If the result is one, it means that there was at least one common field
+     * and all the common fields were the same.
+     * <p>
+     * Similar entries have a score of above 0.8
+     *
+     * @param one The first entry
+     * @param two The second entry
+     * @return number [0,1] 1 representing the same (one potentially having more fields), 0 representing completely different
+     */
+    public double degreeOfSimilarity(final BibEntry one, final BibEntry two) {
+        return one.getFields((f) -> two.getField(f).isPresent())
+                  .stream().mapToDouble((field) -> {
+                    String first = one.getField(field).get();
+                    String second = two.getField(field).get();
+                    int maxLength = Math.max(first.length(), second.length());
+                    Integer levenshteinDistance = LevenshteinDistance.getDefaultInstance().apply(first, second);
+                    return 1 - levenshteinDistance / (double) maxLength;
+                }).average().orElse(0.0);
+    }
 }

jablib/src/main/java/org/jabref/logic/integrity/RefChecker.java

@@ -0,0 +1,244 @@
+package org.jabref.logic.integrity;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+import org.jabref.logic.database.DuplicateCheck;
+import org.jabref.logic.importer.FetcherException;
+import org.jabref.logic.importer.IdBasedFetcher;
+import org.jabref.logic.importer.fetcher.ArXivFetcher;
+import org.jabref.logic.importer.fetcher.CrossRef;
+import org.jabref.logic.importer.fetcher.DoiFetcher;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.BibEntryTypesManager;
+import org.jabref.model.entry.identifier.DOI;
+
+/**
+ * Validates a BibEntry depending on if it
+ * is consistent with the fetched Entry
+ */
+public class RefChecker {
+    DoiFetcher doiFetcher;
+    ArXivFetcher arxivFetcher;
+    CrossRef crossRef;
+    DuplicateCheck duplicateCheck;
+
+    public RefChecker(
+            DoiFetcher doiFetcher,
+            ArXivFetcher arXivFetcher) {
+        this(doiFetcher, arXivFetcher, new CrossRef(), new DuplicateCheck(new BibEntryTypesManager()));
+    }
+
+    public RefChecker(
+            DoiFetcher doiFetcher,
+            ArXivFetcher arXivFetcher,
+            CrossRef crossRef,
+            DuplicateCheck duplicateCheck) {
+        this.doiFetcher = doiFetcher;
+        this.arxivFetcher = arXivFetcher;
+        this.crossRef = crossRef;
+        this.duplicateCheck = duplicateCheck;
+    }
+
+    /**
+     * Tries to find the best reference validity
+     * among current ways. If any of the methods signal
+     * that it is real, it returns early.
+     * <p>
+     * DoiFetcher -> CrossRef -> ArxivFetcher
+     *
+     * @param entry entry checking
+     * @return the reference validity
+     * @throws FetcherException any error from fetchers
+     */
+    public ReferenceValidity referenceValidityOfEntry(BibEntry entry) throws FetcherException {
+        return validityFromDoiFetcher(entry).lazyOr(() ->
+                validityFromCrossRef(entry)
+        ).lazyOr(() -> validityFromArxiv(entry));
+    }
+
+    private ReferenceValidity validityFromFetcher(BibEntry entry, IdBasedFetcher fetcher) throws FetcherException {
+        Optional<DOI> doi = entry.getDOI();
+        if (doi.isEmpty()) {
+            return new Fake();
+        }
+
+        Optional<BibEntry> other = fetcher.performSearchById(doi.get().asString());
+        return other.map(o -> compareReferences(entry, o))
+                    .orElse(new Fake());
+    }
+
+    /**
+     * Tests validity only from the DoiFetcher.
+     *
+     * @param entry the entry
+     * @return the reference validity
+     * @throws FetcherException the fetcher exception
+     */
+    public ReferenceValidity validityFromDoiFetcher(BibEntry entry) throws FetcherException {
+        return validityFromFetcher(entry, doiFetcher);
+    }
+
+    /**
+     * Validity only from the CrossRef and later from the DoiFetcher.
+     *
+     * @param entry the entry
+     * @return the reference validity
+     * @throws FetcherException the fetcher exception
+     */
+    public ReferenceValidity validityFromCrossRef(BibEntry entry) throws FetcherException {
+        Optional<DOI> doiFound = crossRef.findIdentifier(entry);
+
+        if (doiFound.isEmpty()) {
+            return new Fake();
+        } else {
+            DOI doi = doiFound.get();
+            return doiFetcher.performSearchById(doi.asString()).map(
+                    (found) -> compareReferences(entry, found)
+            ).orElse(new Fake());
+        }
+    }
+
+    /**
+     * Validity only from the arxivFetcher.
+     *
+     * @param entry the entry
+     * @return the reference validity
+     * @throws FetcherException the fetcher exception
+     */
+    public ReferenceValidity validityFromArxiv(BibEntry entry) throws FetcherException {
+
+        var m = arxivFetcher.findIdentifier(entry);
+        if (m.isEmpty()) {
+            return new Fake();
+        }
+        return arxivFetcher.performSearchById(m.get().asString()).map(
+                found -> compareReferences(entry, found)
+        ).orElse(new Fake());
+    }
+
+    /**
+     * Takes a list for entries and returns the mapping of them with their corresponding
+     * reference validity.
+     *
+     * @param entries the entries
+     * @return the map
+     * @throws FetcherException the fetcher exception
+     */
+    public Map<BibEntry, ReferenceValidity> validateListOfEntries(List<BibEntry> entries) throws FetcherException {
+
+        Map<BibEntry, ReferenceValidity> entriesToValidity = new HashMap<>();
+        for (BibEntry entry : entries) {
+            entriesToValidity.put(entry, referenceValidityOfEntry(entry));
+        }
+        return entriesToValidity;
+    }
+
+    private ReferenceValidity compareReferences(BibEntry original, BibEntry trueEntry) {
+        double similarity = duplicateCheck.degreeOfSimilarity(original, trueEntry);
+        if (similarity >= 0.999) {
+            return new Real(trueEntry);
+        } else if (similarity > 0.8) {
+            return new Unsure(trueEntry);
+        } else {
+            return new Fake();
+        }
+    }
+
+    @FunctionalInterface
+    private interface ReferenceValiditySupplier {
+        ReferenceValidity get() throws FetcherException;
+    }
+
+    public static abstract sealed class ReferenceValidity permits Real, Unsure, Fake {
+
+        public ReferenceValidity or(ReferenceValidity other) {
+            if (this instanceof Real || other instanceof Fake) {
+                return this;
+            }
+            if (other instanceof Unsure otherUnsure && this instanceof Unsure thisUnsure) {
+                otherUnsure.addAll(thisUnsure);
+            }
+            return other;
+        }
+
+        private ReferenceValidity lazyOr(ReferenceValiditySupplier other) throws FetcherException {
+            if (this instanceof Real) {
+                return this;
+            } else {
+                return or(other.get());
+            }
+        }
+    }
+
+    public static final class Real extends ReferenceValidity {
+        BibEntry matchingReference;
+
+        public Real(BibEntry matchingReference) {
+            this.matchingReference = matchingReference;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            Real real = (Real) o;
+            return Objects.equals(matchingReference, real.matchingReference);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hashCode(matchingReference);
+        }
+    }
+
+    public static final class Unsure extends ReferenceValidity {
+        Set<BibEntry> matchingReferences;
+
+        public Unsure(BibEntry matchingReference) {
+            Set<BibEntry> matchingReferences = new HashSet<>();
+            matchingReferences.add(matchingReference);
+            this.matchingReferences = matchingReferences;
+        }
+
+        void addAll(Unsure other) {
+            this.matchingReferences.addAll(other.matchingReferences);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            Unsure unsure = (Unsure) o;
+            return Objects.equals(matchingReferences, unsure.matchingReferences);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hashCode(matchingReferences);
+        }
+    }
+
+    public static final class Fake extends ReferenceValidity {
+        public boolean equals(Object o) {
+            return o.getClass() == Fake.class;
+        }
+
+        public int hashCode() {
+            return Objects.hashCode(Fake.class);
+        }
+    }
+}

jablib/src/test/java/org/jabref/logic/database/DuplicateCheckTest.java

@@ -7,6 +7,7 @@
 import org.jabref.model.entry.BibEntryTypesManager;
 import org.jabref.model.entry.field.Field;
 import org.jabref.model.entry.field.StandardField;
+import org.jabref.model.entry.field.UnknownField;
 import org.jabref.model.entry.types.StandardEntryType;
 
 import org.junit.jupiter.api.BeforeEach;
@@ -611,4 +612,50 @@ void differentInCollectionWithTheSameISBNAreNotDuplicates() {
 
         assertFalse(duplicateChecker.isDuplicate(entryOne, entryTwo, BibDatabaseMode.BIBTEX));
     }
+
+    @Test
+    void degreeOfSimilarityOfSameEntryIsOne() {
+        assertEquals(1.0, duplicateChecker.degreeOfSimilarity(getSimpleArticle(), getSimpleArticle()));
+        assertEquals(1.0, duplicateChecker.degreeOfSimilarity(getSimpleInCollection(), getSimpleInCollection()));
+    }
+
+    @Test
+    void differentEntriesHaveSmallDegreeOfSimilarity() {
+        assertTrue(0.3 >
+                duplicateChecker.degreeOfSimilarity(
+                        new BibEntry(StandardEntryType.Article)
+                                .withField(StandardField.TITLE, "Some Article"),
+                        new BibEntry(StandardEntryType.InCollection)
+                                .withField(StandardField.TITLE, "Other Collection")
+                )
+        );
+    }
+
+    @Test
+    void entriesWithNoMatchingFieldHaveNoSimilarity() {
+        assertEquals(0.0, duplicateChecker.degreeOfSimilarity(
+                new BibEntry(StandardEntryType.Article)
+                        .withField(StandardField.TITLE, "Some Article"),
+                new BibEntry(StandardEntryType.Article)
+                        .withField(StandardField.AUTHOR, "Some Author")
+        ));
+    }
+
+    @Test
+    void moreFieldsDoesNotAffectTheSimilarity() {
+        assertEquals(1.0, duplicateChecker.degreeOfSimilarity(
+                getSimpleArticle(),
+                getSimpleArticle().withField(new UnknownField("secret"), "Something")
+        ));
+    }
+
+    @Test
+    void similarEntriesHaveAHighDegreeOfSimilarity() {
+        double similarity = duplicateChecker.degreeOfSimilarity(
+                getSimpleArticle().withField(StandardField.YEAR, "2018"),
+                getSimpleArticle()
+        );
+        assertTrue(0.8 < similarity);
+        assertTrue(1.0 > similarity);
+    }
 }