Save&restore search for snapshots contained in composite snapshots. Server parts

georgweiss · georgweiss · commit d9cd17fa5eac · 2025-09-10T12:10:48.000+02:00
diff --git a/services/save-and-restore/src/main/java/org/phoebus/service/saveandrestore/persistence/dao/impl/elasticsearch/CompositeSnapshotDataRepository.java b/services/save-and-restore/src/main/java/org/phoebus/service/saveandrestore/persistence/dao/impl/elasticsearch/CompositeSnapshotDataRepository.java
@@ -22,6 +22,7 @@
 import co.elastic.clients.elasticsearch._types.Refresh;
 import co.elastic.clients.elasticsearch._types.Result;
 import co.elastic.clients.elasticsearch._types.query_dsl.MatchAllQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.MatchPhraseQuery;
 import co.elastic.clients.elasticsearch.core.CountRequest;
 import co.elastic.clients.elasticsearch.core.CountResponse;
 import co.elastic.clients.elasticsearch.core.DeleteByQueryRequest;
@@ -40,12 +41,15 @@
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.phoebus.applications.saveandrestore.model.CompositeSnapshot;
 import org.phoebus.applications.saveandrestore.model.CompositeSnapshotData;
+import org.phoebus.service.saveandrestore.model.ESTreeNode;
+import org.phoebus.service.saveandrestore.search.SearchUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.repository.CrudRepository;
 import org.springframework.http.HttpStatus;
 import org.springframework.stereotype.Repository;
+import org.springframework.util.MultiValueMap;
 import org.springframework.web.server.ResponseStatusException;
 
 import java.io.IOException;
@@ -62,13 +66,17 @@
 @Repository
 public class CompositeSnapshotDataRepository implements CrudRepository<CompositeSnapshotData, String> {
 
+    @SuppressWarnings("unused")
     @Value("${elasticsearch.composite_snapshot_node.index:saveandrestore_composite_snapshot}")
     private String ES_COMPOSITE_SNAPSHOT_INDEX;
 
     @Autowired
     @Qualifier("client")
     ElasticsearchClient client;
 
+    @Autowired
+    private SearchUtil searchUtil;
+
     private final Logger logger = Logger.getLogger(CompositeSnapshotDataRepository.class.getName());
 
     @Override
@@ -234,4 +242,16 @@ public void deleteAll() {
             throw new RuntimeException(e);
         }
     }
+
+    public List<CompositeSnapshotData> containedIn(MultiValueMap<String, String> searchParameters){
+
+        SearchRequest searchRequest = searchUtil.buildSearchRequest(searchParameters);
+        try {
+            SearchResponse<CompositeSnapshotData> response = client.search(searchRequest, CompositeSnapshotData.class);
+            return response.hits().hits().stream().map(Hit::source).collect(Collectors.toList());
+        } catch (IOException e) {
+            logger.log(Level.SEVERE, "Failed to search for referenced snapshot nodes", e);
+            throw new RuntimeException(e);
+        }
+    }
 }
diff --git a/services/save-and-restore/src/main/java/org/phoebus/service/saveandrestore/persistence/dao/impl/elasticsearch/ConfigurationDataRepository.java b/services/save-and-restore/src/main/java/org/phoebus/service/saveandrestore/persistence/dao/impl/elasticsearch/ConfigurationDataRepository.java
@@ -26,6 +26,7 @@
 import co.elastic.clients.elasticsearch.core.search.Hit;
 import co.elastic.clients.transport.endpoints.BooleanResponse;
 import org.phoebus.applications.saveandrestore.model.ConfigurationData;
+import org.phoebus.applications.saveandrestore.model.Node;
 import org.phoebus.service.saveandrestore.search.SearchUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -212,6 +213,5 @@ public List<ConfigurationData> searchOnPvName(MultiValueMap<String, String> sear
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
-
     }
 }
diff --git a/services/save-and-restore/src/main/java/org/phoebus/service/saveandrestore/persistence/dao/impl/elasticsearch/ElasticsearchDAO.java b/services/save-and-restore/src/main/java/org/phoebus/service/saveandrestore/persistence/dao/impl/elasticsearch/ElasticsearchDAO.java
@@ -1289,4 +1289,16 @@ public int compare(String s1, String s2) {
             return index1 - index2;
         }
     }
+
+    public List<Node> containedInCompositeSnapshot(MultiValueMap<String, String> searchParameters){
+        List<CompositeSnapshotData> compositeSnapshotDataList = compositeSnapshotDataRepository.containedIn(searchParameters);
+        List<String> compositeSnapshotIds = compositeSnapshotDataList.stream().map(CompositeSnapshotData::getUniqueId).toList();
+
+        Iterable<ESTreeNode> esTreeNodes = elasticsearchTreeRepository.findAllById(compositeSnapshotIds);
+
+        List<Node> list = new ArrayList<>();
+        esTreeNodes.iterator().forEachRemaining(es -> list.add(es.getNode()));
+
+        return list;
+    }
 }
diff --git a/services/save-and-restore/src/main/java/org/phoebus/service/saveandrestore/search/SearchUtil.java b/services/save-and-restore/src/main/java/org/phoebus/service/saveandrestore/search/SearchUtil.java
@@ -44,6 +44,9 @@ public class SearchUtil {
     @Value("${elasticsearch.configuration_node.index:saveandrestore_configuration}")
     private String ES_CONFIGURATION_INDEX;
     @SuppressWarnings("unused")
+    @Value("${elasticsearch.composite_snapshot_node.index:saveandrestore_composite_snapshot}")
+    private String ES_COMPOSITE_SNAPSHOT_INDEX;
+    @SuppressWarnings("unused")
     @Value("${elasticsearch.result.size.search.default:100}")
     private int defaultSearchSize;
     @SuppressWarnings("unused")
@@ -75,7 +78,8 @@ public SearchRequest buildSearchRequest(MultiValueMap<String, String> searchPara
         LOG.info("  searchParameters: " + searchParameters);
 
         for (Entry<String, List<String>> parameter : searchParameters.entrySet()) {
-            switch (parameter.getKey().strip().toLowerCase()) {
+            String s = parameter.getKey().strip().toLowerCase();
+            switch (s) {
                 case "uniqueid":
                     for (String value : parameter.getValue()) {
                         for (String pattern : value.split("[|,;]")) {
@@ -211,6 +215,8 @@ public SearchRequest buildSearchRequest(MultiValueMap<String, String> searchPara
                         from = Integer.parseInt(maxFrom.get());
                     }
                     break;
+                case "containedin":
+                    return buildSearchRequestForContainedIn(searchParameters);
                 default:
                     // Unsupported search parameters ignored
                     break;
@@ -385,4 +391,46 @@ public List<String> getSearchTerms(String searchQueryTerms) {
         terms.addAll(remaining.stream().filter(t -> t.length() > 0).collect(Collectors.toList()));
         return terms;
     }
+
+    /**
+     * Constructs a {@link SearchRequest} search for composite snapshots containing a node id.
+     * @param searchParameters Map of search parameters where key &quot;containedin&quot; must be present and
+     *                         contain at least one value. Note however that only first value is considered.
+     *                         If no value is specified, an {@link IllegalArgumentException} is thrown.
+     * @return A suitable {@link SearchRequest}
+     */
+    private SearchRequest buildSearchRequestForContainedIn(MultiValueMap<String, String> searchParameters){
+        List<String> value = searchParameters.get("containedin");
+        if(value.isEmpty()){
+            throw new IllegalArgumentException("At least one value must be specified for 'containedin'");
+        }
+        int from = 0;
+        List<String> parameter = searchParameters.get("from");
+        if(parameter != null){
+            Optional<String> maxFrom = parameter.stream().max(Comparator.comparing(Integer::valueOf));
+            if (maxFrom.isPresent()) {
+                from = Integer.parseInt(maxFrom.get());
+            }
+        }
+        int size = maxSearchSize;
+        parameter = searchParameters.get("size");
+        if(parameter != null){
+            Optional<String> maxFrom = parameter.stream().max(Comparator.comparing(Integer::valueOf));
+            if (maxFrom.isPresent()) {
+                size = Integer.parseInt(maxFrom.get());
+            }
+        }
+
+        // Only consider first node id value if multiple are specified
+        String nodeId = value.get(0);
+        int _from = from;
+        int _size = size;
+        MatchPhraseQuery matchPhraseQuery = MatchPhraseQuery.of(m ->
+                m.field("referencedSnapshotNodes").query(nodeId));
+        return SearchRequest.of(s -> s.index(ES_COMPOSITE_SNAPSHOT_INDEX)
+                .query(matchPhraseQuery._toQuery())
+                .timeout("60s")
+                .from(_from)
+                .size(_size));
+    }
 }
diff --git a/services/save-and-restore/src/test/java/org/phoebus/service/saveandrestore/persistence/dao/impl/DAOTestIT.java b/services/save-and-restore/src/test/java/org/phoebus/service/saveandrestore/persistence/dao/impl/DAOTestIT.java
@@ -105,6 +105,9 @@ public class DAOTestIT {
     @Value("${elasticsearch.filter.index:test_saveandrestore_filter}")
     private String ES_FILTER_INDEX;
 
+    @Value("${elasticsearch.composite_snapshot_node.index:test_saveandrestore_composite_snapshot}")
+    private String ES_COMPOSITE_SNAPSHOT_INDEX;
+
     private static Alarm alarm;
     private static Time time;
     private static Display display;
@@ -115,7 +118,6 @@ public void init() {
         alarm = Alarm.of(AlarmSeverity.NONE, AlarmStatus.NONE, "name");
         display = Display.none();
         clearAllData();
-        System.out.println();
     }
 
     @AfterEach
@@ -1826,7 +1828,7 @@ public void testGetAllNodes() {
      */
     private void clearAllData() {
         List<Node> childNodes = nodeDAO.getChildNodes(Node.ROOT_FOLDER_UNIQUE_ID);
-        childNodes.forEach(node -> nodeDAO.deleteNode(node.getUniqueId()));
+        childNodes.forEach(node -> nodeDAO.deleteNodes(List.of(node.getUniqueId())));
         nodeDAO.deleteAllFilters();
     }
 
@@ -1854,6 +1856,17 @@ public void dropIndices() {
         } catch (IOException e) {
             e.printStackTrace();
         }
+
+        try {
+            BooleanResponse exists = client.indices().exists(ExistsRequest.of(e -> e.index(ES_COMPOSITE_SNAPSHOT_INDEX)));
+            if (exists.value()) {
+                client.indices().delete(
+                        DeleteIndexRequest.of(
+                                c -> c.index(ES_COMPOSITE_SNAPSHOT_INDEX)));
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
     }
 
     @Test
@@ -2253,4 +2266,122 @@ public void testSearchForPvs() {
         // No pvs specified -> find all nodes.
         assertEquals(4, searchResult.getHitCount());
     }
+
+    @Test
+    public void testSnapshotContainedInCompositeSnapshot() throws Exception{
+        Node rootNode = nodeDAO.getRootNode();
+        Node folderNode =
+                Node.builder().name("folder").build();
+        folderNode = nodeDAO.createNode(rootNode.getUniqueId(), folderNode);
+
+        //************  Create snapshot1 ************/
+        Node config = Node.builder().nodeType(NodeType.CONFIGURATION).name("My config 1").build();
+        Configuration configuration = new Configuration();
+        configuration.setConfigurationNode(config);
+        ConfigurationData configurationData = new ConfigurationData();
+        configurationData.setPvList(Arrays.asList(ConfigPv.builder().pvName("pv1").build(),
+                ConfigPv.builder().pvName("pv2").build()));
+        configuration.setConfigurationData(configurationData);
+
+        configuration = nodeDAO.createConfiguration(folderNode.getUniqueId(), configuration);
+
+        SnapshotItem item1 = SnapshotItem.builder().configPv(configuration.getConfigurationData().getPvList().get(0))
+                .value(VDouble.of(7.7, alarm, time, display)).build();
+        SnapshotItem item2 = SnapshotItem.builder().configPv(configuration.getConfigurationData().getPvList().get(1))
+                .value(VDouble.of(7.7, alarm, time, display)).build();
+
+        Snapshot snapshot = new Snapshot();
+        snapshot.setSnapshotNode(Node.builder().nodeType(NodeType.SNAPSHOT).name("snapshot name")
+                .description("comment")
+                .userName("user").build());
+        SnapshotData snapshotData = new SnapshotData();
+        snapshotData.setSnapshotItems(Arrays.asList(item1, item2));
+        snapshot.setSnapshotData(snapshotData);
+        Node newSnapshot1 = nodeDAO.createSnapshot(configuration.getConfigurationNode().getUniqueId(), snapshot).getSnapshotNode();
+        //************  End create snapshot1 ************/
+
+        //************  Create snapshot2 ************/
+        Node config2 = Node.builder().nodeType(NodeType.CONFIGURATION).name("My config 2").build();
+        Configuration configuration2 = new Configuration();
+        configuration2.setConfigurationNode(config2);
+        ConfigurationData configurationData2 = new ConfigurationData();
+        configurationData2.setPvList(Arrays.asList(ConfigPv.builder().pvName("pv12").build(),
+                ConfigPv.builder().pvName("pv22").build()));
+        configuration2.setConfigurationData(configurationData2);
+
+        configuration2 = nodeDAO.createConfiguration(folderNode.getUniqueId(), configuration2);
+
+        SnapshotItem item12 = SnapshotItem.builder().configPv(configuration2.getConfigurationData().getPvList().get(0))
+                .value(VDouble.of(7.7, alarm, time, display)).build();
+        SnapshotItem item22 = SnapshotItem.builder().configPv(configuration2.getConfigurationData().getPvList().get(1))
+                .value(VDouble.of(7.7, alarm, time, display)).build();
+
+        Snapshot snapshot2 = new Snapshot();
+        snapshot2.setSnapshotNode(Node.builder().nodeType(NodeType.SNAPSHOT).name("snapshot name 2")
+                .description("comment")
+                .userName("user").build());
+        SnapshotData snapshotData2 = new SnapshotData();
+        snapshotData2.setSnapshotItems(Arrays.asList(item12, item22));
+        snapshot2.setSnapshotData(snapshotData2);
+        Node newSnapshot2 = nodeDAO.createSnapshot(configuration2.getConfigurationNode().getUniqueId(), snapshot2).getSnapshotNode();
+        //************  End create snapshot2 ************/
+
+        //************  Create composite snapshot ************/
+        Node compositeSnapshotNode = Node.builder().name("My composite snapshot").nodeType(NodeType.COMPOSITE_SNAPSHOT).build();
+
+        CompositeSnapshot compositeSnapshot = new CompositeSnapshot();
+        compositeSnapshot.setCompositeSnapshotNode(compositeSnapshotNode);
+
+        CompositeSnapshotData compositeSnapshotData = new CompositeSnapshotData();
+        compositeSnapshotData.setUniqueId(compositeSnapshotNode.getUniqueId());
+
+        compositeSnapshotData.setReferencedSnapshotNodes(Arrays.asList(snapshot.getSnapshotNode().getUniqueId(),
+                snapshot2.getSnapshotNode().getUniqueId()));
+        compositeSnapshot.setCompositeSnapshotData(compositeSnapshotData);
+
+        compositeSnapshot = nodeDAO.createCompositeSnapshot(folderNode.getUniqueId(), compositeSnapshot);
+        //************  End create composite snapshot ************/
+
+        MultiValueMap<String, String> searchParameters = new LinkedMultiValueMap<>();
+        searchParameters.put("containedin", List.of(newSnapshot1.getUniqueId()));
+        searchParameters.put("from", List.of("0"));
+        searchParameters.put("size", List.of("100"));
+
+        List<Node> nodes = nodeDAO.containedInCompositeSnapshot(searchParameters);
+
+        assertEquals(nodes.size(), 1);
+
+        //************  Create another composite snapshot ************/
+        Node compositeSnapshotNode2 = Node.builder().name("My composite snapshot 2").nodeType(NodeType.COMPOSITE_SNAPSHOT).build();
+
+        CompositeSnapshot compositeSnapshot2 = new CompositeSnapshot();
+        compositeSnapshot2.setCompositeSnapshotNode(compositeSnapshotNode2);
+
+        CompositeSnapshotData compositeSnapshotData2 = new CompositeSnapshotData();
+        compositeSnapshotData2.setUniqueId(compositeSnapshotNode2.getUniqueId());
+
+        compositeSnapshotData2.setReferencedSnapshotNodes(Arrays.asList(snapshot.getSnapshotNode().getUniqueId(),
+                snapshot2.getSnapshotNode().getUniqueId()));
+        compositeSnapshot2.setCompositeSnapshotData(compositeSnapshotData2);
+
+        nodeDAO.createCompositeSnapshot(folderNode.getUniqueId(), compositeSnapshot2);
+        //************  End create another composite snapshot ************/
+
+        nodes = nodeDAO.containedInCompositeSnapshot(searchParameters);
+
+        assertEquals(nodes.size(), 2);
+
+        searchParameters.put("containedin", List.of("non-existing"));
+
+        nodes = nodeDAO.containedInCompositeSnapshot(searchParameters);
+
+        assertEquals(nodes.size(), 0);
+
+        searchParameters.put("containedin", Collections.emptyList());
+
+        assertThrows(IllegalArgumentException.class, () -> nodeDAO.containedInCompositeSnapshot(searchParameters));
+
+        nodeDAO.deleteNode(compositeSnapshotNode.getUniqueId());
+        nodeDAO.deleteNode(compositeSnapshotNode2.getUniqueId());
+    }
 }

Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,7 @@`
`26`	`26`	`import co.elastic.clients.elasticsearch.core.search.Hit;`
`27`	`27`	`import co.elastic.clients.transport.endpoints.BooleanResponse;`
`28`	`28`	`import org.phoebus.applications.saveandrestore.model.ConfigurationData;`
	`29`	`+import org.phoebus.applications.saveandrestore.model.Node;`
`29`	`30`	`import org.phoebus.service.saveandrestore.search.SearchUtil;`
`30`	`31`	`import org.springframework.beans.factory.annotation.Autowired;`
`31`	`32`	`import org.springframework.beans.factory.annotation.Qualifier;`
`@@ -212,6 +213,5 @@ public List<ConfigurationData> searchOnPvName(MultiValueMap<String, String> sear`
`212`	`213`	`} catch (IOException e) {`
`213`	`214`	`throw new RuntimeException(e);`
`214`	`215`	`}`
`215`		`-`
`216`	`216`	`}`
`217`	`217`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1289,4 +1289,16 @@ public int compare(String s1, String s2) {`
`1289`	`1289`	`return index1 - index2;`
`1290`	`1290`	`}`
`1291`	`1291`	`}`
	`1292`	`+`
	`1293`	`+ public List<Node> containedInCompositeSnapshot(MultiValueMap<String, String> searchParameters){`
	`1294`	`+ List<CompositeSnapshotData> compositeSnapshotDataList = compositeSnapshotDataRepository.containedIn(searchParameters);`
	`1295`	`+ List<String> compositeSnapshotIds = compositeSnapshotDataList.stream().map(CompositeSnapshotData::getUniqueId).toList();`
	`1296`	`+`
	`1297`	`+ Iterable<ESTreeNode> esTreeNodes = elasticsearchTreeRepository.findAllById(compositeSnapshotIds);`
	`1298`	`+`
	`1299`	`+ List<Node> list = new ArrayList<>();`
	`1300`	`+ esTreeNodes.iterator().forEachRemaining(es -> list.add(es.getNode()));`
	`1301`	`+`
	`1302`	`+ return list;`
	`1303`	`+ }`
`1292`	`1304`	`}`