New study.yml format (JabRef#13844)

* Reformatted database variable to catalogue

* Renamed StudyDatabase.java to StudyCatalog and updated its usage in other classes

* Created an abstract StudyYamlMigrator class as a template for study.yml migrations

* Created an implementation of the abstract Migrator class for migrations from the current study.yml version to the new one

* Created a StudyMetadata class to handle the metadata part of the study.yml file

* Created a new class called StudyYamlService for handling the migration and parsing of study.yml if needed

* StudyYamlMigrator no longer uses child classes in constructor in order to avoid deadlock

* Ran openrewrite and changed the jetbrains annotations in StudyYamlV1Migrator to JSpecify annotations

* Changed StudyYamlParserTest.java to use the new StudyYamlParserTest instead of StudyYamlParser

* Changed the usage of the word catalogue to catalog to be consistent with the rest of the codebase

* Added functions for adding a metadata field to the study.yml

* Added unit tests for MigratorV1, removed the study type field as it was not needed

* Changed database variable to catalog and an immutable map is returned from getCatalogSpecific form StudyQuery

* Reformatted files

* Removed studytype parameter in StudyMetadata

* yaml mapper variable in StudyYamlMigrator is now a static field

* Changed null handling on some variables to use Optional

* Changed the use of StringBuilder to StringJoiner in StudyYamlV1Migrator

* Added Optionals to various functions

* Refactored the tests for StudyYamlV1Migrator to use Optionals correctly

* Refactored the tests for StudyYamlV1Migrator to use Optionals correctly

* Refactored the tests for StudyYamlV1Migrator to use Optionals correctly

* Refactored StudyQuery to not use collections import

* Disabled the requirement for handlers for java8 optionals in StudyYamlParser

* Reverted the disabling of the requirement for handlers for optionals in StudyYamlParser and removed Optionals in StudyQuery

* Changed the usage of Optionals in Study and readded Optionals on StudyQuery

* Added new dependency jackson datatype jdk 8 for serializing Optionals

* Refactored Study and StudyQuery to use more Optionals, StudyYamlParser should support proper deserialization of Optionals

* Added annotations for Json deserialization to avoid conflicts

* getCatalogSpecificOptional function is ignored when deserializing Json

* updated StudyQuery equals method for compatibility with testing

* Update jablib/src/main/java/org/jabref/logic/crawler/StudyYamlV1Migrator.java

Co-authored-by: Carl Christian Snethlage <[email protected]>

* Reverted the update to instanceof check because of unsafe type conversion

* Added nonnull annotations and changed the metadata field to not use Optional

* Added nonnull annotations to getVersion function from StudyYamlV1Migrator

* Changed the suppress warning annotation to the whole function in StudyYamlV1Migrator

* fix version and remove unused stuff

* readd

---------

Co-authored-by: Carl Christian Snethlage <[email protected]>
Co-authored-by: Christoph <[email protected]>

Author: 3 people

jabgui/src/main/java/org/jabref/gui/slr/EditExistingStudyAction.java

@@ -8,7 +8,7 @@
 import org.jabref.gui.actions.ActionHelper;
 import org.jabref.gui.actions.SimpleCommand;
 import org.jabref.logic.crawler.StudyRepository;
-import org.jabref.logic.crawler.StudyYamlParser;
+import org.jabref.logic.crawler.StudyYamlService;
 import org.jabref.logic.l10n.Localization;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.study.Study;
@@ -51,7 +51,7 @@ public void execute() {
 
         Study study;
         try {
-            study = new StudyYamlParser().parseStudyYamlFile(studyDirectory.resolve(StudyRepository.STUDY_DEFINITION_FILE_NAME));
+            study = new StudyYamlService().parseStudyYamlFile(studyDirectory.resolve(StudyRepository.STUDY_DEFINITION_FILE_NAME));
         } catch (IOException e) {
             dialogService.showErrorDialogAndWait(Localization.lang("Error opening file"), e);
             return;

jabgui/src/main/java/org/jabref/gui/slr/ManageStudyDefinitionViewModel.java

@@ -17,7 +17,7 @@
 import org.jabref.gui.DialogService;
 import org.jabref.gui.WorkspacePreferences;
 import org.jabref.logic.crawler.StudyRepository;
-import org.jabref.logic.crawler.StudyYamlParser;
+import org.jabref.logic.crawler.StudyYamlService;
 import org.jabref.logic.git.GitHandler;
 import org.jabref.logic.importer.ImportFormatPreferences;
 import org.jabref.logic.importer.ImporterPreferences;
@@ -30,7 +30,7 @@
 import org.jabref.logic.importer.fetcher.SpringerNatureWebFetcher;
 import org.jabref.logic.l10n.Localization;
 import org.jabref.model.study.Study;
-import org.jabref.model.study.StudyDatabase;
+import org.jabref.model.study.StudyCatalog;
 import org.jabref.model.study.StudyQuery;
 
 import org.eclipse.jgit.api.errors.GitAPIException;
@@ -103,15 +103,15 @@ public ManageStudyDefinitionViewModel(@NonNull Study study,
         title.setValue(study.getTitle());
         researchQuestions.addAll(study.getResearchQuestions());
         queries.addAll(study.getQueries().stream().map(StudyQuery::getQuery).toList());
-        List<StudyDatabase> studyDatabases = study.getDatabases();
+        List<StudyCatalog> studyCatalogs = study.getCatalogs();
         databases.addAll(WebFetchers.getSearchBasedFetchers(importFormatPreferences, importerPreferences)
                                     .stream()
                                     .map(SearchBasedFetcher::getName)
                                     // The user wants to select specific fetchers
                                     // The fetcher summarizing ALL fetchers can be emulated by selecting ALL fetchers (which happens rarely when doing an SLR)
                                     .filter(name -> !CompositeSearchBasedFetcher.FETCHER_NAME.equals(name))
                                     .map(name -> {
-                                        boolean enabled = studyDatabases.contains(new StudyDatabase(name, true));
+                                        boolean enabled = studyCatalogs.contains(new StudyCatalog(name, true));
                                         return new StudyCatalogItem(name, enabled);
                                     })
                                     .toList());
@@ -172,7 +172,7 @@ public SlrStudyAndDirectory saveStudy() {
                 title.getValueSafe(),
                 researchQuestions,
                 queries.stream().map(StudyQuery::new).collect(Collectors.toList()),
-                databases.stream().map(studyDatabaseItem -> new StudyDatabase(studyDatabaseItem.getName(), studyDatabaseItem.isEnabled())).filter(StudyDatabase::isEnabled).collect(Collectors.toList()));
+                databases.stream().map(studyDatabaseItem -> new StudyCatalog(studyDatabaseItem.getName(), studyDatabaseItem.isEnabled())).filter(StudyCatalog::isEnabled).collect(Collectors.toList()));
         Path studyDirectory;
         final String studyDirectoryAsString = directory.getValueSafe();
         try {
@@ -185,7 +185,7 @@ public SlrStudyAndDirectory saveStudy() {
         }
         Path studyDefinitionFile = studyDirectory.resolve(StudyRepository.STUDY_DEFINITION_FILE_NAME);
         try {
-            new StudyYamlParser().writeStudyYamlFile(study, studyDefinitionFile);
+            new StudyYamlService().writeStudyYamlFile(study, studyDefinitionFile);
         } catch (IOException e) {
             LOGGER.error("Could not write study file {}", studyDefinitionFile, e);
             dialogService.notify(Localization.lang("Please enter a valid file path.") +

jabgui/src/main/java/org/jabref/gui/slr/StartNewStudyAction.java

@@ -10,7 +10,7 @@
 import org.jabref.gui.StateManager;
 import org.jabref.gui.importer.actions.OpenDatabaseAction;
 import org.jabref.logic.crawler.StudyRepository;
-import org.jabref.logic.crawler.StudyYamlParser;
+import org.jabref.logic.crawler.StudyYamlService;
 import org.jabref.logic.git.GitHandler;
 import org.jabref.logic.l10n.Localization;
 import org.jabref.logic.preferences.CliPreferences;
@@ -57,8 +57,8 @@ public StartNewStudyAction(LibraryTabContainer tabContainer,
 
     @Override
     protected void crawlPreparation(Path studyRepositoryRoot) throws IOException, GitAPIException {
-        StudyYamlParser studyYAMLParser = new StudyYamlParser();
-        studyYAMLParser.writeStudyYamlFile(newStudy, studyRepositoryRoot.resolve(StudyRepository.STUDY_DEFINITION_FILE_NAME));
+        StudyYamlService studyYamlService = new StudyYamlService();
+        studyYamlService.writeStudyYamlFile(newStudy, studyRepositoryRoot.resolve(StudyRepository.STUDY_DEFINITION_FILE_NAME));
 
         // When execution reaches this point, the user created a new study.
         // The GitHandler is already called to initialize the repository with one single commit "Initial commit".

jabgui/src/main/java/org/jabref/gui/slr/StudyCatalogItem.java

@@ -7,12 +7,12 @@
 import javafx.beans.property.SimpleStringProperty;
 import javafx.beans.property.StringProperty;
 
-import org.jabref.model.study.StudyDatabase;
+import org.jabref.model.study.StudyCatalog;
 
 import org.jspecify.annotations.NonNull;
 
 /**
- * View representation of {@link StudyDatabase}
+ * View representation of {@link StudyCatalog}
  */
 public class StudyCatalogItem {
     private final StringProperty name;

jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java

@@ -8,7 +8,7 @@
 import org.jabref.logic.importer.ImportFormatPreferences;
 import org.jabref.logic.importer.ImporterPreferences;
 import org.jabref.model.study.Study;
-import org.jabref.model.study.StudyDatabase;
+import org.jabref.model.study.StudyCatalog;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -34,7 +34,7 @@ void setUp() {
     }
 
     @Test
-    void emptyStudyConstructorFillsDatabasesCorrectly() {
+    void emptyStudyConstructorFillsCatalogsCorrectly() {
         ManageStudyDefinitionViewModel manageStudyDefinitionViewModel = new ManageStudyDefinitionViewModel(importFormatPreferences, importerPreferences, workspacePreferences, dialogService);
         assertEquals(List.of(
                 new StudyCatalogItem("ACM Portal", true),
@@ -65,7 +65,7 @@ void emptyStudyConstructorFillsDatabasesCorrectly() {
     }
 
     @Test
-    void studyConstructorFillsDatabasesCorrectly(@TempDir Path tempDir) {
+    void studyConstructorFillsCatalogsCorrectly(@TempDir Path tempDir) {
         ManageStudyDefinitionViewModel manageStudyDefinitionViewModel = getManageStudyDefinitionViewModel(tempDir);
         assertEquals(List.of(
                 new StudyCatalogItem("ACM Portal", true),
@@ -96,14 +96,14 @@ void studyConstructorFillsDatabasesCorrectly(@TempDir Path tempDir) {
     }
 
     private ManageStudyDefinitionViewModel getManageStudyDefinitionViewModel(Path tempDir) {
-        List<StudyDatabase> databases = List.of(
-                new StudyDatabase("ACM Portal", true));
+        List<StudyCatalog> catalogs = List.of(
+                new StudyCatalog("ACM Portal", true));
         Study study = new Study(
                 List.of("Name"),
                 "title",
                 List.of("Q1"),
                 List.of(),
-                databases
+                catalogs
         );
         return new ManageStudyDefinitionViewModel(
                 study,

jablib/build.gradle.kts

@@ -107,6 +107,7 @@ dependencies {
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310")
     // TODO: Somwewhere we get a warning: unknown enum constant Id.CLASS reason: class file for com.fasterxml.jackson.annotation.JsonTypeInfo$Id not found
     // implementation("com.fasterxml.jackson.core:jackson-annotations:2.19.1")
+    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jdk8")
 
     implementation("com.fasterxml:aalto-xml")
 

jablib/src/main/java/module-info.java

@@ -260,5 +260,6 @@
     requires org.jooq.jool;
     requires org.libreoffice.uno;
     requires transitive org.jspecify;
+    requires com.fasterxml.jackson.datatype.jdk8;
     // endregion
 }

jablib/src/main/java/org/jabref/logic/crawler/StudyCatalogToFetcherConverter.java

@@ -9,17 +9,17 @@
 import org.jabref.logic.importer.ImporterPreferences;
 import org.jabref.logic.importer.SearchBasedFetcher;
 import org.jabref.logic.importer.WebFetchers;
-import org.jabref.model.study.StudyDatabase;
+import org.jabref.model.study.StudyCatalog;
 
 /**
  * Converts library entries from the given study into their corresponding fetchers.
  */
 class StudyCatalogToFetcherConverter {
-    private final List<StudyDatabase> libraryEntries;
+    private final List<StudyCatalog> libraryEntries;
     private final ImportFormatPreferences importFormatPreferences;
     private final ImporterPreferences importerPreferences;
 
-    public StudyCatalogToFetcherConverter(List<StudyDatabase> libraryEntries,
+    public StudyCatalogToFetcherConverter(List<StudyCatalog> libraryEntries,
                                           ImportFormatPreferences importFormatPreferences,
                                           ImporterPreferences importerPreferences) {
         this.libraryEntries = libraryEntries;
@@ -44,7 +44,7 @@ public List<SearchBasedFetcher> getActiveFetchers() {
      * @param libraryEntries List of entries
      * @return List of fetcher instances
      */
-    private List<SearchBasedFetcher> getFetchersFromLibraryEntries(List<StudyDatabase> libraryEntries) {
+    private List<SearchBasedFetcher> getFetchersFromLibraryEntries(List<StudyCatalog> libraryEntries) {
         return libraryEntries.parallelStream()
                              .map(this::createFetcherFromLibraryEntry)
                              .filter(Objects::nonNull)
@@ -54,12 +54,12 @@ private List<SearchBasedFetcher> getFetchersFromLibraryEntries(List<StudyDatabas
     /**
      * Transforms a library entry into a SearchBasedFetcher instance. This only works if the library entry specifies a supported fetcher.
      *
-     * @param studyDatabase the entry that will be converted
+     * @param studyCatalog the entry that will be converted
      * @return An instance of the fetcher defined by the library entry.
      */
-    private SearchBasedFetcher createFetcherFromLibraryEntry(StudyDatabase studyDatabase) {
+    private SearchBasedFetcher createFetcherFromLibraryEntry(StudyCatalog studyCatalog) {
         Set<SearchBasedFetcher> searchBasedFetchers = WebFetchers.getSearchBasedFetchers(importFormatPreferences, importerPreferences);
-        String libraryNameFromFetcher = studyDatabase.getName();
+        String libraryNameFromFetcher = studyCatalog.getName();
         return searchBasedFetchers.stream()
                                   .filter(searchBasedFetcher -> searchBasedFetcher.getName().equalsIgnoreCase(libraryNameFromFetcher))
                                   .findAny()

jablib/src/main/java/org/jabref/logic/crawler/StudyRepository.java

@@ -34,7 +34,7 @@
 import org.jabref.model.study.FetchResult;
 import org.jabref.model.study.QueryResult;
 import org.jabref.model.study.Study;
-import org.jabref.model.study.StudyDatabase;
+import org.jabref.model.study.StudyCatalog;
 import org.jabref.model.study.StudyQuery;
 import org.jabref.model.util.FileUpdateMonitor;
 
@@ -116,9 +116,9 @@ public StudyRepository(Path pathToRepository,
 
             gitHandler.checkoutBranch(SEARCH_BRANCH);
             // If study definition does not exist on this branch or was changed on work branch, copy it from work
-            boolean studyDefinitionDoesNotExistOrChanged = !(Files.exists(studyDefinitionFile) && new StudyYamlParser().parseStudyYamlFile(studyDefinitionFile).equals(study));
+            boolean studyDefinitionDoesNotExistOrChanged = !(Files.exists(studyDefinitionFile) && new StudyYamlService().parseStudyYamlFile(studyDefinitionFile).equals(study));
             if (studyDefinitionDoesNotExistOrChanged) {
-                new StudyYamlParser().writeStudyYamlFile(study, studyDefinitionFile);
+                new StudyYamlService().writeStudyYamlFile(study, studyDefinitionFile);
             }
             setUpRepositoryStructureForQueriesAndFetchers();
             gitHandler.createCommitOnCurrentBranch(updateRepositoryStructureMessage, false);
@@ -175,7 +175,7 @@ public BibDatabaseContext getStudyResultEntries() throws IOException {
      * @throws IOException Problem opening the input stream.
      */
     private Study parseStudyFile() throws IOException {
-        return new StudyYamlParser().parseStudyYamlFile(studyDefinitionFile);
+        return new StudyYamlService().parseStudyYamlFile(studyDefinitionFile);
     }
 
     /**
@@ -196,10 +196,10 @@ public List<String> getSearchQueryStrings() {
      * @return List of BibEntries of type Library
      * @throws IllegalArgumentException If a transformation from Library entry to LibraryDefinition fails
      */
-    public List<StudyDatabase> getActiveLibraryEntries() throws IllegalArgumentException {
-        return study.getDatabases()
+    public List<StudyCatalog> getActiveLibraryEntries() throws IllegalArgumentException {
+        return study.getCatalogs()
                     .parallelStream()
-                    .filter(StudyDatabase::isEnabled)
+                    .filter(StudyCatalog::isEnabled)
                     .collect(Collectors.toList());
     }
 

jablib/src/main/java/org/jabref/logic/crawler/StudyYamlMigrator.java

@@ -0,0 +1,100 @@
+package org.jabref.logic.crawler;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.jabref.model.study.Study;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Abstract base class for migrating study.yml files between versions.
+ * Provides common functionality for version detection
+ * and handles the migration process
+ */
+
+public abstract class StudyYamlMigrator {
+    protected static final String CURRENT_VERSION = "2.0";
+    private static final Logger LOGGER = LoggerFactory.getLogger(StudyYamlMigrator.class);
+    private static final ObjectMapper YAML_MAPPER = new ObjectMapper(new YAMLFactory());
+
+    public static StudyYamlMigrator getMigratorForVersion(String version) {
+        return switch (version) {
+            case "1.0",
+                 "unknown" ->
+                    new StudyYamlV1Migrator();
+            default -> {
+                LOGGER.warn("Unknown version {}, using V1 migrator", version);
+                yield new StudyYamlV1Migrator();
+            }
+        };
+    }
+
+    /**
+     * Main entry point for migration. Detects version and delegates to appropriate migrator.
+     */
+    public static Study migrateStudyYamlFile(Path studyYamlFile) throws IOException {
+        String version = detectVersion(studyYamlFile);
+
+        if (CURRENT_VERSION.equals(version)) {
+            // Already current version, read the file normally
+            try (InputStream fileInputStream = Files.newInputStream(studyYamlFile)) {
+                return YAML_MAPPER.readValue(fileInputStream, Study.class);
+            }
+        }
+
+        LOGGER.info("Migrating study.yml from version {} to {}", version, CURRENT_VERSION);
+
+        StudyYamlMigrator migrator = getMigratorForVersion(version);
+        Study migratedStudy = migrator.migrate(studyYamlFile);
+
+        LOGGER.info("Successfully migrated study.yml to version {}", CURRENT_VERSION);
+        return migratedStudy;
+    }
+
+    /**
+     * Template method that subclasses must implement for specific version migration
+     */
+    protected abstract Study migrate(Path studyYamlFile) throws IOException;
+
+    /**
+     * Returns the version this migrator handles
+     */
+    protected abstract String getSourceVersion();
+
+    /**
+     * Returns the target version this migrator produces
+     */
+    protected String getTargetVersion() {
+        return CURRENT_VERSION;
+    }
+
+    /**
+     * Detects the version of a study.yml file
+     */
+    private static String detectVersion(Path studyYamlFile) throws IOException {
+        ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory());
+
+        try (InputStream fileInputStream = Files.newInputStream(studyYamlFile)) {
+            JsonNode rootNode = yamlMapper.readTree(fileInputStream);
+            JsonNode versionNode = rootNode.get("version");
+
+            if (versionNode != null) {
+                return versionNode.asText();
+            }
+
+            // If no version field, check for old structure indicators
+            if (rootNode.has("databases") && !rootNode.has("catalogues")) {
+                return "1.0"; // Old format
+            }
+
+            return "unknown";
+        }
+    }
+}