Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
d578f69
Port DuplicateCheck.similarity to StringSimilarity and add tests
TheYorouzoya Aug 15, 2025
022bcd4
Add ICORE2023 Rank Data To Resources
TheYorouzoya Aug 20, 2025
5bacb29
Add Core Classes For ICORE Rank Lookup
TheYorouzoya Aug 22, 2025
e035e0e
Integrate ICORE Rank Lookup Feature Into The GUI
TheYorouzoya Aug 26, 2025
3a36afe
Merge branch 'main' into add-ICORE-ranking-support
TheYorouzoya Aug 26, 2025
a7b1b94
Add Missing Localization Keys and Fix Broken Test
TheYorouzoya Aug 27, 2025
33ed0db
Use List.of() and fix grammar
koppor Aug 28, 2025
806309e
Reorder fields
koppor Aug 28, 2025
5330632
Fix unsupported operation exception
koppor Aug 28, 2025
b092aa9
Rename field
koppor Aug 28, 2025
52d3acd
Merge branch 'main' into add-ICORE-ranking-support
koppor Aug 28, 2025
75d2b47
Hotfix: calling of publish.yml
koppor Aug 28, 2025
45a1d10
Port DuplicateCheck.similarity to StringSimilarity and add tests
TheYorouzoya Aug 15, 2025
290cf6c
Add ICORE2023 Rank Data To Resources
TheYorouzoya Aug 20, 2025
93da09f
Add Core Classes For ICORE Rank Lookup
TheYorouzoya Aug 22, 2025
cbd4dda
Fix Merge Conflict From Upstream Fetch
TheYorouzoya Aug 26, 2025
2a1a73e
Add Missing Localization Keys and Fix Broken Test
TheYorouzoya Aug 27, 2025
6747ad1
Merged changes to FieldFactory and StandardField
TheYorouzoya Aug 28, 2025
ff5c144
Add Minor Fixes, Documentation, and Refactor
TheYorouzoya Aug 28, 2025
89c6600
Revert "Hotfix: calling of publish.yml"
TheYorouzoya Aug 28, 2025
c63e2da
Remove duplicate line in CHANGELOG
TheYorouzoya Aug 28, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 2 additions & 28 deletions jablib/src/main/java/org/jabref/logic/database/DuplicateCheck.java
Original file line number Diff line number Diff line change
Expand Up @@ -289,9 +289,10 @@ public static double correlateByWords(final String s1, final String s2) {
final String[] w1 = s1.split("\\s");
final String[] w2 = s2.split("\\s");
final int n = Math.min(w1.length, w2.length);
final StringSimilarity match = new StringSimilarity();
int misses = 0;
for (int i = 0; i < n; i++) {
double corr = similarity(w1[i], w2[i]);
double corr = match.similarity(w1[i], w2[i]);
if (corr < 0.75) {
misses++;
}
Expand All @@ -300,33 +301,6 @@ public static double correlateByWords(final String s1, final String s2) {
return 1 - missRate;
}

/**
* Calculates the similarity (a number within 0 and 1) between two strings.
* http://stackoverflow.com/questions/955110/similarity-string-comparison-in-java
*/
private static double similarity(final String first, final String second) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice find :)

final String longer;
final String shorter;

if (first.length() < second.length()) {
longer = second;
shorter = first;
} else {
longer = first;
shorter = second;
}

final int longerLength = longer.length();
// both strings are zero length
if (longerLength == 0) {
return 1.0;
}
final double distanceIgnoredCase = new StringSimilarity().editDistanceIgnoreCase(longer, shorter);
final double similarity = (longerLength - distanceIgnoredCase) / longerLength;
LOGGER.trace("Longer string: {} Shorter string: {} Similarity: {}", longer, shorter, similarity);
return similarity;
}

/**
* Checks if the two entries represent the same publication.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,12 @@
import java.util.Locale;

import info.debatty.java.stringsimilarity.Levenshtein;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class StringSimilarity {
private static final Logger LOGGER = LoggerFactory.getLogger(StringSimilarity.class);

private final Levenshtein METRIC_DISTANCE = new Levenshtein();
// edit distance threshold for entry title comparison
private final int METRIC_THRESHOLD = 4;
Expand All @@ -24,4 +28,31 @@ public double editDistanceIgnoreCase(String a, String b) {
// TODO: Locale is dependent on the language of the strings. English is a good denominator.
return METRIC_DISTANCE.distance(a.toLowerCase(Locale.ENGLISH), b.toLowerCase(Locale.ENGLISH));
}

/**
* Calculates the similarity (a number within 0 and 1) between two strings.
* http://stackoverflow.com/questions/955110/similarity-string-comparison-in-java
*/
public double similarity(final String first, final String second) {
final String longer;
final String shorter;

if (first.length() < second.length()) {
longer = second;
shorter = first;
} else {
longer = first;
shorter = second;
}

final int longerLength = longer.length();
// both strings are zero length
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This comment is trivial and can be derived directly from the code condition (longerLength == 0). It should be removed as it doesn't add new information.

if (longerLength == 0) {
Comment on lines +49 to +50
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Comment is redundant as it simply restates what the code clearly shows. The comment doesn't provide additional information or reasoning.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The comment was added there by the original author of the code, I've merely ported it with the necessary modifications. That said, I will contest this review as the lines

final int longerLength = longer.length();
// both strings are zero length
if (longerLength == 0) {
    return 1.0;
}

do not explicitly state, on their own, that the two input strings are equal. Further, the comment is present in the original Stack Overflow post here. That being said, if you're adamant about it, I don't mind changing this.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Take the bot meanings with care, they are not always that good

return 1.0;
}
final double distanceIgnoredCase = editDistanceIgnoreCase(longer, shorter);
final double similarity = (longerLength - distanceIgnoredCase) / longerLength;
LOGGER.trace("Longer string: {} Shorter string: {} Similarity: {}", longer, shorter, similarity);
return similarity;
}
}
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
package org.jabref.logic.util.strings;

import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

class StringSimilarityTest {
private final double EPSILON_SIMILARITY = 0.00001;

private StringSimilarity similarityChecker = new StringSimilarity();

Expand All @@ -27,4 +30,64 @@ class StringSimilarityTest {
void stringSimilarity(String a, String b, String expectedResult) {
assertEquals(Boolean.valueOf(expectedResult), similarityChecker.isSimilar(a, b));
}

@Test
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the test

Please, convert to @ParamterizedTest - and if possible to @CsvSource

void similarityReturnsOneForExactStrings() {
String a = "abcdef";
String b = "abcdef";
double expectedResult = 1.0;
double similarity = similarityChecker.similarity(a, b);

assertEquals(expectedResult, similarity, EPSILON_SIMILARITY);
}

@Test
void similarityReturnsZeroForNonMatchingStrings() {
String a = "abcdef";
String b = "uvwxyz";
double expectedResult = 0.0;
double similarity = similarityChecker.similarity(a, b);

assertEquals(expectedResult, similarity, EPSILON_SIMILARITY);
}

@Test
void similarityReturnsValueBetweenZeroAndOneForSimilarStrings() {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if this test is doing really the expected thing -- check for the expected value directly.

String a = "abc";
String b = "abcdefgh";
double exactMatch = 1.0;
double similarity = similarityChecker.similarity(a, b);

assertTrue(similarity >= EPSILON_SIMILARITY && similarity < exactMatch);
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using assertTrue with a boolean condition instead of asserting the actual contents. Should compare the actual similarity value with expected bounds using assertEquals.

}

@Test
void similarityReturnsOneForEmptyStrings() {
String a = "";
String b = "";
double expectedResult = 1.0;
double similarity = similarityChecker.similarity(a, b);

assertEquals(expectedResult, similarity, EPSILON_SIMILARITY);
}

@Test
void similarityReturnsZeroWhenOneStringIsEmpty() {
String a = "abcdef";
String b = "";
double expectedResult = 0.0;
double similarity = similarityChecker.similarity(a, b);

assertEquals(expectedResult, similarity, EPSILON_SIMILARITY);
}

@Test
void similarityIsCaseInsensitive() {
String a = "abcdef";
String b = "ABCDEF";
double expectedResult = 1.0;
double similarity = similarityChecker.similarity(a, b);

assertEquals(expectedResult, similarity, EPSILON_SIMILARITY);
}
}
Loading