Merge pull request #679 from Esri/Stac_Fixes

Fixed Dcat request

Author: mhogeweg

geoportal-search/src/main/resources/gs/base/Request.js

@@ -63,14 +63,10 @@
 
     getSortOptions: {writable:true,value:function() {
       // &sortField=title,modified&sortOrder=desc,asc // Portal syntax
-      // &sort={"title.keyword":{"order":"asc","unmapped_type":"keyword"}} // Elasticsearch syntax
+      // &sort=title:desc&sort=modified:asc // Elasticsearch syntax
       // &sortBy=title:D,modified:A // CSW syntax
       var idx, sortField, sortOrder, sortOptions = null;
       var values = this.getParameterValues("sort");
-      if(values!== null)
-	  {
-    	  return values; //For ElasticSearch, it will be handled in ElasticTarget
-	  }
       if (values === null || values.length === 0) {
         values = this.getParameterValues("sortBy");
       }

geoportal-search/src/main/resources/gs/context/nashorn/NashornContext.js

@@ -219,8 +219,7 @@
     },
 
     sendHttpRequest: function(url, data, dataContentType, options) {
-     // print("url",url);
-     // print("data",data);
+      //print("url",url);
       var result = null;
       var br = null, br2 = null, wr = null;
       var sw = new java.io.StringWriter();
@@ -230,7 +229,7 @@
         var u = new java.net.URL(url);
 
 		var client = com.esri.geoportal.lib.elastic.http.ElasticClient.newClient();;
-        //print(u);
+        //print("data ",data);
         if(options && options.useHttps)
         	{        	
         	 var ssl_ctx = javax.net.ssl.SSLContext.getInstance("TLS");

geoportal-search/src/main/resources/gs/target/elastic/ElasticTarget.js

@@ -357,26 +357,64 @@
     }},
 
     prepareSort: {writable:true,value:function(task,targetRequest) {
-        var sortables = this.schema.sortables;
-        if (!sortables) return;    
-
-        var sort = [], sortOptions = task.request.getSortOptions();
-        //print("sortOptions",sortOptions);
-        if (Array.isArray(sortOptions)) {    	 
-        	sortOptions.forEach(function(sortOption){
-          	//print("sortOption ",sortOption);
-  			 
-	  		try {
-	  			sort = JSON.parse(sortOption);			 
-	  		} catch (e) {
-	  			sort = sortOption;
-	  		}
-        	});
-        }
-        if (sort) {
-          targetRequest.searchCriteria["sort"] = sort;
+      var sortables = this.schema.sortables;
+      if (!sortables) return;
+      print("sortables ",JSON.stringify(sortables));
+      var getField = function(v) {
+        v = v.toLowerCase();
+        for (var k in sortables) {
+          if (sortables.hasOwnProperty(k)) {
+            if (v === k.toLowerCase()) {
+              return sortables[k];
+            }
+          }
         }
-      }},
+        return null;
+      };
+
+      var sort = [], sortOptions = task.request.getSortOptions();
+      //print("ElasticTarget sortOptions ",JSON.stringify(sortOptions));
+      if (Array.isArray(sortOptions)) {
+        sortOptions.forEach(function(sortOption){
+          var field = getField(sortOption.field);
+          //print("sort field ",field);
+          if (typeof field === "string" && field.length > 0) {
+            var option = {};
+            if (sortOption.order === "asc") {
+				if(field ==='title')
+				{
+					//print("sort field title asc");
+					option['title.keyword']={"order": "asc", "unmapped_type": "keyword"}
+				}
+				else{
+					//print("sort field not title");
+					option[field] = "asc";
+				}
+					
+              
+            } else if (sortOption.order === "desc") {
+				if(field ==='title')
+				{
+					//print("sort field title desc");
+					option['title.keyword']={"order": "desc", "unmapped_type": "keyword"}
+				}
+				else{
+					//print("sort field not title desc");
+					option[field] = "desc";
+				}				
+              
+            } else {
+              option = field;
+            }
+            sort = option;
+          }
+        });
+      }
+      if (sort) {
+        targetRequest.searchCriteria["sort"] = sort;
+        //console.log("final sort ", JSON.stringify(targetRequest.searchCriteria["sort"]));
+      }
+    }},
 
     prepareTimePeriod: {writable:true,value:function(task,targetRequest) {
       var period = task.request.getTimePeriod();

geoportal/src/main/java/com/esri/geoportal/dcat/DcatBuilder.java

@@ -161,22 +161,7 @@ private String getSelfInfo() {
     try {
       JsonObjectBuilder access = Json.createObjectBuilder();
       access.add("supportsApprovalStatus",com.esri.geoportal.context.GeoportalContext.getInstance().getSupportsApprovalStatus());
-      access.add("supportsGroupBasedAccess",com.esri.geoportal.context.GeoportalContext.getInstance().getSupportsGroupBasedAccess());    
-      com.esri.geoportal.context.AppUser user = null;
-      if (user != null && user.getUsername() != null) {
-        access.add("username",user.getUsername());
-        access.add("isAdmin",user.isAdmin());
-        if (com.esri.geoportal.context.GeoportalContext.getInstance().getSupportsGroupBasedAccess()) {
-          JsonArrayBuilder jsaGroups = Json.createArrayBuilder();
-          List<com.esri.geoportal.base.security.Group> groups = user.getGroups();
-          if (groups != null) {
-            for (com.esri.geoportal.base.security.Group group: groups) {
-              jsaGroups.add(group.id);
-            }         
-          }
-          access.add("groups",jsaGroups);
-        }
-      }
+      access.add("supportsGroupBasedAccess",com.esri.geoportal.context.GeoportalContext.getInstance().getSupportsGroupBasedAccess()); 
       elastic.add("access",access);
     } catch (Throwable t) {
       LOGGER.warn(String.format("Warning getting self info."), t);

geoportal/src/main/java/com/esri/geoportal/search/STACService.java

@@ -34,6 +34,8 @@
 import java.util.logging.Level;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import javax.json.JsonArray;
 import javax.json.JsonObject;
@@ -70,8 +72,10 @@
 import com.esri.geoportal.service.stac.Collection;
 import com.esri.geoportal.service.stac.GeometryServiceClient;
 import com.esri.geoportal.service.stac.StacContext;
+import com.jayway.jsonpath.Configuration;
 import com.jayway.jsonpath.DocumentContext;
 import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.Option;
 
 import net.minidev.json.JSONArray;
 import net.minidev.json.JSONObject;
@@ -495,7 +499,8 @@ public Response getItems(@Context HttpServletRequest hsr,
 			else
 				response = client.sendGet(url);
 
-			responseJSON = this.prepareResponse(response, hsr, bbox, limit, datetime, null, null, "metadataItems", collectionId,null);      
+			responseJSON = this.prepareResponse(response, hsr, bbox, limit, datetime, null, null, "metadataItems", collectionId,null,
+					null,null,outCRS, null, null);      
 
       // if reprojecting STAC geometries is supported and a
       // geometry service has been configured, try projecting 
@@ -835,7 +840,7 @@ public Response search(@Context HttpServletRequest hsr, @QueryParam("limit") int
 			{
 				responseJSON = this.prepareResponse(response, hsr, bbox, limit, datetime, 
                         idList, intersects, "search", 
-                        listOfCollections,null);
+                        listOfCollections,null, updated, created, outCRS, itemStatus, filter);
 
 				// if re-projecting STAC geometries is supported and a 
 				// geometry service has been configured, try projecting from internal CRS (4326) to requested outCRS
@@ -884,7 +889,8 @@ public Response search(@Context HttpServletRequest hsr, @RequestBody String body
 		JsonArray bboxJsonArr = (requestPayload.containsKey("bbox") ? requestPayload.getJsonArray("bbox") : null);
 		JsonArray idArr = (requestPayload.containsKey("ids") ? requestPayload.getJsonArray("ids") : null);
 		String outCRS = (requestPayload.containsKey("outCRS") ? requestPayload.getString("outCRS") : null);
-		
+		search_after = (requestPayload.containsKey("search_after") ? requestPayload.getString("search_after") : search_after);
+				
 		JsonArray collectionArr = (requestPayload.containsKey("collections")
 				? requestPayload.getJsonArray("collections")
 				: null);
@@ -971,14 +977,15 @@ public Response search(@Context HttpServletRequest hsr, @RequestBody String body
 		        queryMap.put("filterClause", filterClause); //filterQry);
 		    }
 
-		 //Search request with outCRS is valid, if only one collection in collections param, otherwise 400
-	       if ((outCRS != null) &&  collectionArr!=null && collectionArr.size()>1)
-	       {
+		  //Search request with outCRS is valid, if only one collection in collections param, otherwise 400
+      if ((outCRS != null) &&  collectionArr!=null && collectionArr.size()>1)
+      {
 			  status = Response.Status.BAD_REQUEST;    				
 			  responseJSON = this.generateResponse("400", "Only one collection can be included in search param if search param includes outCRS ",null);
 			  return Response.status(status).header("Content-Type", "application/geo+json").entity(responseJSON).build();
 
-	       }
+      }
+      
 			//Adding one extra so that next page can be figured out
 			url = url + "/_search?size=" + (limit+1);
 			query = StacHelper.prepareSearchQuery(queryMap, search_after);
@@ -1000,15 +1007,15 @@ public Response search(@Context HttpServletRequest hsr, @RequestBody String body
 		      // if re-projecting STAC geometries is supported and a
 		      // geometry service has been configured, try projecting 
 		      // from internal CRS (4326) to requested outCRS
-			      if ((outCRS != null) && ("true".equals(sc.isCanStacGeomTransform())) && (!gc.getGeometryService().isEmpty())) {
-				        LOGGER.debug("outCRS = " + outCRS + " - " + gc.getGeometryService());
-				
-				        JSONObject projectedResponseObj = projectSearchResults(responseJSON, "4326", outCRS);
-				        responseJSON = projectedResponseObj.toString();             
-				        
-				        // done
-				        LOGGER.debug("Project response -> " + responseJSON);
-			      }
+          if ((outCRS != null) && ("true".equals(sc.isCanStacGeomTransform())) && (!gc.getGeometryService().isEmpty())) {
+              LOGGER.debug("outCRS = " + outCRS + " - " + gc.getGeometryService());
+
+              JSONObject projectedResponseObj = projectSearchResults(responseJSON, "4326", outCRS);
+              responseJSON = projectedResponseObj.toString();             
+
+              // done
+              LOGGER.debug("Project response -> " + responseJSON);
+          }
 			}
 
 		} catch (InvalidParameterException e) {
@@ -1819,9 +1826,16 @@ private String prepareResponseSingleItem(String searchRes, String itemFileString
 		finalResponse = finalResponse.replaceAll("\\{collectionId\\}", collectionId);
 		return finalResponse;
 	}
+	
+	private String prepareResponse(String searchRes, HttpServletRequest hsr, String bbox, int limit, String datetime,
+			String ids, String intersects, String requestType, String collectionId, String body ) {
+			return this.prepareResponse(searchRes, hsr, bbox, limit, datetime, ids, intersects, requestType, collectionId, body, 
+					null, null, null, null, null);
+	}
 
 	private String prepareResponse(String searchRes, HttpServletRequest hsr, String bbox, int limit, String datetime,
-			String ids, String intersects, String requestType, String collectionId, String body) {
+			String ids, String intersects, String requestType, String collectionId, String body,
+			String updated,String created,String outCrs,String status,String filter) {
 
 		int numberMatched;		
 		net.minidev.json.JSONArray items;
@@ -1898,18 +1912,15 @@ private String prepareResponse(String searchRes, HttpServletRequest hsr, String
 			}
 			String urlparam = "";
 			if (requestType.equalsIgnoreCase("searchPost")) {
-				JSONObject bodyObj =new JSONObject();
-				// In post request, search_after will be part of request body				
-				if (body != null) 
-				{
-					bodyObj = (JSONObject) JSONValue.parse(body);
-					if(search_after != null && search_after.length()>0)
-						bodyObj.appendField("search_after", search_after);
-				}					
+				// Add search_after in urlparam
+				urlparam = (search_after != null ? "?search_after=" + search_after : "");        
+				
+				JSONObject bodyObj =new JSONObject();								
 				if(nextLink)
 					linksContext.set("$.searchItem.links[1].body",(body != null ? bodyObj : ""));
 
 			} else {
+				// GET request, set everything as request parameters
 				if(nextLink)
 				{
 					linksContext.delete("$.searchItem.links[1].body");
@@ -1922,6 +1933,11 @@ private String prepareResponse(String searchRes, HttpServletRequest hsr, String
 						+ (search_after != null ? "&search_after=" + search_after : "")
 						+ (encodedIntersect != null ? "&intersects=" + encodedIntersect : "")
 						+ (ids != null ? "&ids=" + ids : "")
+						+ (updated != null ? "&updated=" + updated : "")
+						+ (created != null ? "&created=" + created : "")
+						+ (outCrs != null ? "&outCRS=" + outCrs : "")
+						+ (status != null ? "&status=" + status : "")
+						+ (filter != null ? "&filter=" + filter : "")
 						+((requestType.startsWith("search")) && collectionId != null ? "&collections=" + collectionId : "");
 			}
 			if (requestType.startsWith("metadataItems"))
@@ -1963,7 +1979,8 @@ private boolean populateFeature(DocumentContext featureContext, DocumentContext
 			featureContext.set("$.featurePropPath.id", searchItemCtx.read(val));
 
 			val = featureContext.read("$.featurePropPath.collection");
-			featureContext.set("$.featurePropPath.collection", searchItemCtx.read(val));
+			String collectionId = searchItemCtx.read(val).toString();
+			featureContext.set("$.featurePropPath.collection", collectionId);
 
 			// add bbox, geometry
 			this.setBbox(searchItemCtx, featureContext);
@@ -2027,26 +2044,69 @@ private boolean populateFeature(DocumentContext featureContext, DocumentContext
 				LOGGER.trace("No assets ($._source.assets) in this Stac record with id: " + recordId);
 
 			}
-
-			// Iterate properties, skip property if it is not available			
+			
+			//#680 
+			//fill item with collection properties when not available in item, item property overrides default set in collection	
+			Set<String> collectionPropKeySet = null;
+			JSONObject collectionPropObj = null;
+			ArrayList <String> propToBeAddedFromCollectionList = new ArrayList<String>();
+				  
+			    Collection collection = new Collection(collectionId);
+			    if(collection.getProperties() != null)
+			    {
+			    	collectionPropObj = collection.getProperties();
+				    if (collectionPropObj != null) {
+				        collectionPropKeySet =  collectionPropObj.keySet(); 
+				        //Create a combined set of property keys from item and collection
+				        Set<String> combinedPropSet = Stream.concat(propObjKeys.stream(), collectionPropKeySet.stream())
+			                     .collect(Collectors.toSet()); 
+						 propObjKeys = combinedPropSet;
+				    }	
+			 }			
+			
+			// Iterate properties, skip property if it is not available in item and collection			
 			for (String propKey : propObjKeys) {
 				try {
-					propKeyVal = String.valueOf(propObj.get(propKey));
-					// If it is a json path, set values from search result
-					if (propKeyVal.startsWith("$")) {
-						if (searchItemCtx.read(propKeyVal) != null) {
-							featureContext.set("$.featurePropPath.properties." + propKey,
-									searchItemCtx.read(propKeyVal));
+					//it is item prop, try to fill from searched item
+					if(propObj.containsKey(propKey))
+					{
+						propKeyVal = String.valueOf(propObj.get(propKey));
+						// If it is a json path, set values from search result
+						if (propKeyVal.startsWith("$")) {
+							if (searchItemCtx.read(propKeyVal) != null) {
+								featureContext.set("$.featurePropPath.properties." + propKey,
+										searchItemCtx.read(propKeyVal));
+							}
 						}
 					}
+					//Not an Item prop, Collection prop
+					else
+					{
+						propToBeAddedFromCollectionList.add(propKey);
+					}
+					
 				} catch (Exception e) {
-					// If json path not found or error in any property, remove this property in the
-					// end.
-					// if removed here, concurrentModificationException
-					propToBeRemovedList.add("$.featurePropPath.properties." + propKey);
-					LOGGER.trace("key: " + propKey + " could not be added. Reason : " + e.getMessage());
+					//item did not have this data, so check if it is available in collection prop. This will be added later from collection
+					if(collectionPropObj != null && collectionPropObj.containsKey(propKey))
+					{
+						propToBeAddedFromCollectionList.add(propKey);
+					}
+					// If json path not found in searchItemCtx.read(propKeyVal) or error in any property, remove this property in the
+					// end. if removed here, concurrentModificationException
+					else 	
+					{
+						propToBeRemovedList.add("$.featurePropPath.properties." + propKey);
+						LOGGER.trace("key: " + propKey + " could not be added. Reason : " + e.getMessage());
+					}
 				}
 			}
+			//Read updated item property object again and add keys from collection
+			HashMap<String, String> updatedPropObj = featureContext.read("$.featurePropPath.properties");
+			for(String prop: propToBeAddedFromCollectionList)
+			{
+				updatedPropObj.put(prop, collectionPropObj.getAsString(prop));
+			}
+			featureContext.set("$.featurePropPath.properties",updatedPropObj);
 
 			String linkSelfHref = featureContext.read("$.featurePropPath.links[0].href");
 			linkSelfHref = linkSelfHref.replaceAll("\\{itemId\\}", featureContext.read("$.featurePropPath.id").toString());