Merge pull request #44 from HumanBrainProject/develop

Develop

Author: olinux

src/main/java/org/humanbrainproject/knowledgegraph/KgQueryApplication.java

@@ -5,14 +5,20 @@
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.web.context.request.RequestContextListener;
 
 
 @SpringBootApplication
 @ComponentScan("org.humanbrainproject.knowledgegraph")
 @NoTests(NoTests.NO_LOGIC)
+@EnableCaching
+@EnableScheduling
+@EnableAsync
 public class KgQueryApplication {
 
 	public static void main(String[] args) {
@@ -77,4 +83,5 @@ public ArangoConnection createReleasedDb() {
 		return new ArangoConnection("kg_released", false);
 	}
 
+
 }

src/main/java/org/humanbrainproject/knowledgegraph/commons/api/ParameterConstants.java

@@ -1,6 +1,7 @@
 package org.humanbrainproject.knowledgegraph.commons.api;
 
 import org.humanbrainproject.knowledgegraph.annotations.NoTests;
+import org.humanbrainproject.knowledgegraph.query.entity.fieldFilter.ExampleValues;
 
 @NoTests(NoTests.NO_LOGIC)
 public class ParameterConstants {
@@ -34,6 +35,6 @@ public class ParameterConstants {
     public static final String RESTRICTED_ORGANIZATION_DOC = "Restrict the results to explicitly defined organizations - the main use case is if a user with broad access permissions wants to simulate restricted read access (e.g. the indexing functionality of the KG search UI)";
     public static final String SEARCH_DOC = "A search string checking for instances with names which match this term (includes wildcards)";
 
-    public static final String BOUNDING_BOX_DOC = "A minimal bounding box - if defined, only results which are spatially anchored and are part of the given region are returned. Needs to be defined in combination with referenceSpace";
-    public static final String REFERENCE_SPACE_DOC = "The reference space for the defined minimal bounding box (mbb)";
+    public static final String BOUNDING_BOX_DOC = "A minimal bounding box - if defined, only results which are spatially anchored and are part of the given region are returned. Follow the pattern \""+ ExampleValues.MBB_EXAMPLE+"\"";
+
 }

src/main/java/org/humanbrainproject/knowledgegraph/commons/authorization/control/AuthorizationContext.java

@@ -18,6 +18,8 @@ public interface AuthorizationContext {
 
     Set<String> getReadableOrganizations();
 
+    Set<String> getReadableOrganizations(Credential credential, List<String> whitelistedOrganizations);
+
     Set<String> getReadableOrganizations(List<String> whitelistedOrganizations);
 
     void populateAuthorizationContext(String authorizationToken);

src/main/java/org/humanbrainproject/knowledgegraph/commons/authorization/control/DefaultAuthorizationContext.java

@@ -55,6 +55,11 @@ public Set<String> getReadableOrganizations(){
         return authorizationController.getReadableOrganizations(getCredential(), null);
     }
 
+    @Override
+    public Set<String> getReadableOrganizations(Credential credential, List<String> whitelistedOrganizations){
+        return authorizationController.getReadableOrganizations(credential, whitelistedOrganizations);
+    }
+
     @Override
     public Set<String> getReadableOrganizations(List<String> whitelistedOrganizations) {
         return authorizationController.getReadableOrganizations(getCredential(), whitelistedOrganizations);

src/main/java/org/humanbrainproject/knowledgegraph/commons/jsonld/control/JsonLdStandardization.java

@@ -172,7 +172,7 @@ public <T> T filterKeysByVocabBlacklists(T input) {
      * @throws IOException
      */
     public Map fullyQualify(String jsonPayload) {
-        return fullyQualify(jsonTransformer.parseToMap(jsonPayload));
+        return jsonPayload == null ? null : fullyQualify(jsonTransformer.parseToMap(jsonPayload));
     }
 
     /**

src/main/java/org/humanbrainproject/knowledgegraph/commons/propertyGraph/arango/control/ArangoInternalRepository.java

@@ -12,12 +12,14 @@
 import org.humanbrainproject.knowledgegraph.commons.propertyGraph.arango.control.query.ArangoQueryFactory;
 import org.humanbrainproject.knowledgegraph.commons.propertyGraph.arango.entity.ArangoCollectionReference;
 import org.humanbrainproject.knowledgegraph.commons.propertyGraph.arango.entity.ArangoDocumentReference;
+import org.humanbrainproject.knowledgegraph.commons.propertyGraph.arango.exceptions.StoredQueryNotFoundException;
 import org.humanbrainproject.knowledgegraph.commons.vocabulary.ArangoVocabulary;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import javax.ws.rs.NotFoundException;
 import java.util.List;
 import java.util.Map;
 
@@ -126,4 +128,18 @@ public boolean doesDocumentExist(ArangoCollectionReference collectionReference,
         }
         return false;
     }
+
+    public void removeInternalDocument(ArangoDocumentReference document) throws IllegalAccessException, StoredQueryNotFoundException {
+        String userId = authorizationContext.getUserId();
+        if(userId==null){
+            throw new IllegalAccessException("You have to be authenticated if you want to execute this operation");
+        }
+        ArangoDatabase db = getDB();
+        ArangoCollection collection = db.collection(document.getCollection().getName());
+        if ( collection.exists() && collection.documentExists(document.getKey())) {
+            collection.deleteDocument(document.getKey());
+        } else {
+            throw new StoredQueryNotFoundException("Query not found");
+        }
+    }
 }

src/main/java/org/humanbrainproject/knowledgegraph/commons/propertyGraph/arango/control/ArangoNativeRepository.java

@@ -21,6 +21,8 @@
 import org.springframework.stereotype.Component;
 
 import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 @Component

src/main/java/org/humanbrainproject/knowledgegraph/commons/propertyGraph/arango/control/ArangoToNexusLookupMap.java

@@ -3,12 +3,10 @@
 import org.humanbrainproject.knowledgegraph.annotations.ToBeTested;
 import org.humanbrainproject.knowledgegraph.commons.propertyGraph.arango.entity.ArangoCollectionReference;
 import org.humanbrainproject.knowledgegraph.indexing.entity.nexus.NexusSchemaReference;
-import org.humanbrainproject.knowledgegraph.structure.boundary.Structure;
+import org.humanbrainproject.knowledgegraph.structure.boundary.StructureFromNexus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.InitializingBean;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 import java.util.Collections;
@@ -17,30 +15,16 @@
 
 @Component
 @ToBeTested
-public class ArangoToNexusLookupMap implements InitializingBean {
+public class ArangoToNexusLookupMap {
 
     @Autowired
-    Structure structure;
-
-    @Value("${org.humanbrainproject.knowledgegraph.cache.populate}")
-    boolean populateCache = true;
-
+    StructureFromNexus structureFromNexus;
 
     protected Logger logger = LoggerFactory.getLogger(ArangoToNexusLookupMap.class);
 
-    @Override
-    public void afterPropertiesSet() {
-        try {
-            if(populateCache) {
-                refetch();
-            }
-        }
-        catch(Exception e){
-            logger.error("Failed to load schema cache", e);
-        }
-    }
-
     private static final Map<String, NexusSchemaReference> schemaReferenceMap = Collections.synchronizedMap(new HashMap<>());
+    private static boolean initialFetch = false;
+
 
     public static void addToSchemaReferenceMap(ArangoCollectionReference arangoName, NexusSchemaReference schemaReference){
         if(!schemaReferenceMap.containsKey(arangoName.getName())) {
@@ -59,18 +43,18 @@ public synchronized NexusSchemaReference getNexusSchema(ArangoCollectionReferenc
     }
 
     private void refetch(){
+        initialFetch = true;
         logger.info("Start fetching schemas - cache population in process");
-        structure.getAllSchemasInMainSpace().forEach(ArangoCollectionReference::fromNexusSchemaReference);
+        structureFromNexus.getAllSchemasInMainSpace().forEach(ArangoCollectionReference::fromNexusSchemaReference);
         logger.info("Done fetching schemas - the cache is populated");
     }
 
+
     public Map<String, NexusSchemaReference> getLookupTable(boolean refetch){
-        if(schemaReferenceMap.isEmpty() || refetch){
+        if(!initialFetch || schemaReferenceMap.isEmpty() || refetch){
             refetch();
         }
         return schemaReferenceMap;
     }
 
-
-
 }

src/main/java/org/humanbrainproject/knowledgegraph/commons/propertyGraph/arango/control/aql/AQL.java

@@ -0,0 +1,155 @@
+package org.humanbrainproject.knowledgegraph.commons.propertyGraph.arango.control.aql;
+
+import org.apache.commons.text.StringSubstitutor;
+import org.humanbrainproject.knowledgegraph.annotations.NoTests;
+import org.humanbrainproject.knowledgegraph.annotations.ToBeTested;
+import org.humanbrainproject.knowledgegraph.commons.propertyGraph.arango.entity.ArangoAlias;
+import org.humanbrainproject.knowledgegraph.commons.vocabulary.ArangoVocabulary;
+
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+@ToBeTested(easy = true)
+public class AQL {
+
+    public final String WHITELIST_ALIAS = "whitelist";
+
+    final Map<String, String> parameters = new LinkedHashMap<>();
+    private StringBuilder query = new StringBuilder();
+    private int indent = 0;
+
+    /**
+     * With applying this method to a string, you state that you have checked, the provided string does not contain any unchecked "dynamic" part (such as user inputs).
+     *
+     * The string should consist only of string constants, other trusted values and concatenations out of them. Dynamic parts can be defined with property placeholders - such as ${foo} since they are checked before insertion for their trustworthiness.
+     *
+     * Please be careful when executing your checks! Passing non-validated strings to this method can introduce vulnerabilities to the system!!!
+     */
+    public final static TrustedAqlValue trust(String trustedString){
+        return new TrustedAqlValue(trustedString);
+    }
+
+
+    @NoTests(NoTests.TRIVIAL)
+    public AQL indent() {
+        this.indent++;
+        return this;
+    }
+
+    @NoTests(NoTests.TRIVIAL)
+    public AQL outdent() {
+        this.indent = Math.max(this.indent - 1, 0);
+        return this;
+    }
+
+    public AQL setParameter(String key, String value) {
+        setTrustedParameter(key, preventAqlInjection(value));
+        return this;
+    }
+
+    /**
+     * Use with caution! The passed parameter will not be further checked for AQL injection but rather immediately added to the query!
+     */
+    public AQL setTrustedParameter(String key, TrustedAqlValue trustedAqlValue) {
+        parameters.put(key, trustedAqlValue!=null ? trustedAqlValue.getValue() : null);
+        return this;
+    }
+
+    @NoTests(NoTests.TRIVIAL)
+    public AQL addLine(TrustedAqlValue queryLine) {
+        if(queryLine!=null) {
+            query.append(createIndent());
+            add(queryLine);
+            query.append('\n');
+        }
+        return this;
+    }
+
+    public AQL add(TrustedAqlValue trustedAqlValue){
+        if(trustedAqlValue!=null) {
+            query.append(trustedAqlValue.getValue());
+        }
+        return this;
+    }
+
+    public AQL addComma(){
+        query.append(", ");
+        return this;
+    }
+
+
+    public TrustedAqlValue listCollections(Set<String> values) {
+        return listValuesWithQuote('`', values);
+    }
+
+    public TrustedAqlValue listValues(Set<String> values) {
+        return listValuesWithQuote('"', values);
+    }
+
+    public TrustedAqlValue listFields(Set<String> values){
+        return listValuesWithQuote(null, values);
+    }
+
+
+    private TrustedAqlValue listValuesWithQuote(Character quote, Set<String> values) {
+        if(values!=null && values.size()>0){
+            String q = quote!=null ? String.valueOf(quote) : "";
+            return new TrustedAqlValue(q+String.join(q + "," + q, values.stream().map(v -> preventAqlInjection(v).getValue()).collect(Collectors.toSet()))+q);
+        }
+        else{
+            return new TrustedAqlValue("");
+        }
+    }
+
+    public TrustedAqlValue preventAqlInjection(String value){
+        return value!=null ? new TrustedAqlValue(value.replaceAll("[^A-Za-z0-9\\-_:.#/@]", "")) : null;
+    }
+
+    public TrustedAqlValue preventAqlInjectionForSearchQuery(String value){
+        String f = value.replaceAll("[^\\sA-Za-z0-9\\-_:.#/@]", "");
+        return new TrustedAqlValue(f);
+    }
+
+    public TrustedAqlValue generateSearchTermQuery(TrustedAqlValue value){
+        String f = String.join(" ", Arrays.stream(value.getValue().split(" ")).map(el -> String.format("%%%s%%", el.trim().toLowerCase())).collect(Collectors.toList()));
+        if(f.isEmpty()){
+            f = "%";
+        }
+        return new TrustedAqlValue(f);
+    }
+
+
+    public TrustedAqlValue build() {
+        return new TrustedAqlValue(StringSubstitutor.replace(query.toString(), parameters));
+    }
+
+    private String createIndent() {
+        StringBuilder sb = new StringBuilder();
+        for (int i = 0; i < this.indent; i++) {
+            sb.append("   ");
+        }
+        return sb.toString();
+    }
+
+    public AQL addDocumentFilter(TrustedAqlValue documentAlias) {
+        addLine(new TrustedAqlValue("FILTER "+documentAlias.getValue()+" != NULL"));
+        return this;
+    }
+
+    public AQL addDocumentFilter(ArangoAlias alias){
+        return addDocumentFilter(preventAqlInjection(alias.getArangoDocName()));
+    }
+    public AQL addDocumentFilterWithWhitelistFilter(ArangoAlias alias) {
+        return addDocumentFilterWithWhitelistFilter(preventAqlInjection(alias.getArangoDocName()));
+    }
+
+
+    public AQL addDocumentFilterWithWhitelistFilter(TrustedAqlValue documentAlias) {
+        addDocumentFilter(documentAlias);
+        addLine(trust("FILTER "+documentAlias.getValue()+"."+ ArangoVocabulary.PERMISSION_GROUP+" IN "+WHITELIST_ALIAS));
+        return this;
+    }
+}

src/main/java/org/humanbrainproject/knowledgegraph/commons/propertyGraph/arango/control/aql/AuthorizedArangoQuery.java

@@ -1,29 +1,29 @@
 package org.humanbrainproject.knowledgegraph.commons.propertyGraph.arango.control.aql;
 
 import org.humanbrainproject.knowledgegraph.annotations.ToBeTested;
+import org.humanbrainproject.knowledgegraph.commons.vocabulary.ArangoVocabulary;
 
 import java.util.Set;
 
 @ToBeTested(easy = true)
-public class AuthorizedArangoQuery extends UnauthorizedArangoQuery {
+public class AuthorizedArangoQuery extends AQL {
 
-    public final String WHITELIST_ALIAS = "whitelist";
 
     public AuthorizedArangoQuery(Set<String> permissionGroupsWithReadAccess){
         this(permissionGroupsWithReadAccess, false);
     }
 
     public AuthorizedArangoQuery(Set<String> permissionGroupsWithReadAccess, boolean subQuery) {
         if(!subQuery){
-            addLine("LET "+WHITELIST_ALIAS+"=[${"+WHITELIST_ALIAS+"}]");
+            addLine(trust("LET "+WHITELIST_ALIAS+"=[${"+WHITELIST_ALIAS+"}]"));
             setTrustedParameter(WHITELIST_ALIAS, listValues(permissionGroupsWithReadAccess));
         }
     }
 
     @Override
     public AuthorizedArangoQuery addDocumentFilter(TrustedAqlValue documentAlias) {
         super.addDocumentFilter(documentAlias);
-        addLine("FILTER "+documentAlias.getValue()+"._permissionGroup IN "+WHITELIST_ALIAS);
+        addLine(trust("FILTER "+documentAlias.getValue()+"."+ ArangoVocabulary.PERMISSION_GROUP+" IN "+WHITELIST_ALIAS));
         return this;
     }
 }