Skip to content

Commit d54b991

Browse files
holmbergiusholmbergius
authored
Merge pull request #993 from WildMeOrg/872_matchingset_opensearch
matchingSet built using OpenSearch
2 parents 065b67a + 2d740db commit d54b991

File tree

8 files changed

+338
-319
lines changed

8 files changed

+338
-319
lines changed

‎src/main/java/org/ecocean/Annotation.java‎

Lines changed: 250 additions & 241 deletions
Large diffs are not rendered by default.

‎src/main/java/org/ecocean/Encounter.java‎

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4363,20 +4363,35 @@ public void run() {
43634363
bgShepherd.beginDBTransaction();
43644364
try {
43654365
Encounter enc = bgShepherd.getEncounter(encId);
4366-
if ((enc == null) || !enc.hasMarkedIndividual()) {
4366+
if ((enc == null) || (!enc.hasMarkedIndividual() && !enc.hasAnnotations())) {
43674367
// bgShepherd.rollbackAndClose();
43684368
executor.shutdown();
43694369
return;
43704370
}
43714371
MarkedIndividual indiv = enc.getIndividual();
4372-
System.out.println("opensearchIndexDeep() background indexing indiv " +
4373-
indiv.getId() + " via enc " + encId);
4374-
try {
4375-
indiv.opensearchIndex();
4376-
} catch (Exception ex) {
4377-
System.out.println("opensearchIndexDeep() background indexing " +
4378-
indiv.getId() + " FAILED: " + ex.toString());
4379-
ex.printStackTrace();
4372+
if (indiv != null) {
4373+
System.out.println("opensearchIndexDeep() background indexing indiv " +
4374+
indiv.getId() + " via enc " + encId);
4375+
try {
4376+
indiv.opensearchIndex();
4377+
} catch (Exception ex) {
4378+
System.out.println("opensearchIndexDeep() background indexing " +
4379+
indiv.getId() + " FAILED: " + ex.toString());
4380+
ex.printStackTrace();
4381+
}
4382+
}
4383+
if (enc.hasAnnotations()) {
4384+
for (Annotation ann : enc.getAnnotations()) {
4385+
System.out.println("opensearchIndexDeep() background indexing annot " +
4386+
ann.getId() + " via enc " + encId);
4387+
try {
4388+
ann.opensearchIndex();
4389+
} catch (Exception ex) {
4390+
System.out.println("opensearchIndexDeep() background indexing " +
4391+
ann.getId() + " FAILED: " + ex.toString());
4392+
ex.printStackTrace();
4393+
}
4394+
}
43804395
}
43814396
} catch (Exception e) {
43824397
System.out.println("opensearchIndexDeep() backgrounding Encounter " + encId +
@@ -4390,7 +4405,7 @@ public void run() {
43904405
executor.shutdown();
43914406
}
43924407
};
4393-
System.out.println("opensearchIndexDeep() begin backgrounding indiv for " + this);
4408+
System.out.println("opensearchIndexDeep() begin backgrounding for " + this);
43944409
executor.execute(rn);
43954410
System.out.println("opensearchIndexDeep() [foreground] finished for Encounter " + encId);
43964411
}

‎src/main/java/org/ecocean/OpenSearch.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,25 @@ public JSONObject queryPit(String indexName, final JSONObject query, int numFrom
397397
return new JSONObject(rtn);
398398
}
399399

400+
// just return the actual hit results
401+
// note: each object in the array has _id but actual doc is in _source!!
402+
public static JSONArray getHits(JSONObject queryResults) {
403+
JSONArray failed = new JSONArray();
404+
405+
if (queryResults == null) return failed;
406+
JSONObject outerHits = queryResults.optJSONObject("hits");
407+
if (outerHits == null) {
408+
System.out.println("could not find (outer) hits");
409+
return failed;
410+
}
411+
JSONArray hits = outerHits.optJSONArray("hits");
412+
if (hits == null) {
413+
System.out.println("could not find hits");
414+
return failed;
415+
}
416+
return hits;
417+
}
418+
400419
// https://opensearch.org/docs/2.3/opensearch/search/paginate/
401420
public JSONObject queryRawScroll(String indexName, final JSONObject query, int pageSize)
402421
throws IOException {

‎src/main/java/org/ecocean/Shepherd.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2063,6 +2063,26 @@ public Iterator getAllAnnotationsNoQuery() {
20632063
}
20642064
}
20652065

2066+
// note: where clause can also contain " ORDER BY xxx"
2067+
public Iterator getAnnotationsFilter(String jdoWhereClause) {
2068+
Query query = null;
2069+
2070+
try {
2071+
query = pm.newQuery("SELECT FROM org.ecocean.Annotation WHERE " + jdoWhereClause);
2072+
Collection c = (Collection)(query.execute());
2073+
List list = new ArrayList(c);
2074+
Iterator it = list.iterator();
2075+
query.closeAll();
2076+
return it;
2077+
} catch (Exception npe) {
2078+
System.out.println(
2079+
"Error encountered when trying to execute getAllAnnotationsFilter. Returning a null iterator.");
2080+
npe.printStackTrace();
2081+
if (query != null) query.closeAll();
2082+
return null;
2083+
}
2084+
}
2085+
20662086
public Iterator<Encounter> getAllAnnotations(String order) {
20672087
Extent extClass = pm.getExtent(Annotation.class, true);
20682088
Query q = pm.newQuery(extClass);

‎src/main/java/org/ecocean/api/SearchApi.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response)
5353
} else if ((searchQueryId == null) && !OpenSearch.isValidIndexName(indexName)) {
5454
response.setStatus(404);
5555
res.put("error", "unknown index");
56+
} else if ("annotation".equals(indexName) && !currentUser.isAdmin(myShepherd)) {
57+
// per discussion with jh today, api exposure of annotations admin-only currently
58+
response.setStatus(403);
59+
res.put("error", 403);
5660
} else if ((query == null) && !"POST".equals(request.getMethod())) {
5761
response.setStatus(405);
5862
res.put("error", "method not allowed");

‎src/main/java/org/ecocean/ia/plugin/WildbookIAM.java‎

Lines changed: 18 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@
1111
import javax.servlet.ServletContextEvent;
1212
import org.apache.commons.lang3.builder.ToStringBuilder;
1313
import org.ecocean.acm.AcmUtil;
14+
import org.ecocean.Annotation;
1415
import org.ecocean.cache.CachedQuery;
1516
import org.ecocean.cache.QueryCache;
1617
import org.ecocean.cache.QueryCacheFactory;
17-
import org.ecocean.Annotation;
1818
import org.ecocean.ia.IA;
1919
import org.ecocean.ia.Task;
2020
import org.ecocean.media.*;
@@ -82,48 +82,6 @@ public void prime() {
8282
IBEISIA.setIAPrimed(true);
8383
}
8484

85-
public void _OLD_prime() {
86-
IA.log("INFO: WildbookIAM.prime(" + this.context + ") called");
87-
IBEISIA.setIAPrimed(false);
88-
if (!isEnabled()) return;
89-
final List<String> iaAnnotIds = iaAnnotationIds();
90-
final List<String> iaImageIds = iaImageIds();
91-
Runnable r = new Runnable() {
92-
public void run() {
93-
Shepherd myShepherd = new Shepherd(context);
94-
myShepherd.setAction("WildbookIAM.prime");
95-
myShepherd.beginDBTransaction();
96-
ArrayList<Annotation> matchingSet = Annotation.getAllMatchAgainstTrue(myShepherd);
97-
ArrayList<Annotation> sendAnns = new ArrayList<Annotation>();
98-
ArrayList<MediaAsset> mas = new ArrayList<MediaAsset>();
99-
for (Annotation ann : matchingSet) {
100-
if (iaAnnotIds.contains(ann.getAcmId())) continue; // no need to send
101-
sendAnns.add(ann);
102-
MediaAsset ma = ann.getDerivedMediaAsset();
103-
if (ma == null) ma = ann.getMediaAsset();
104-
if (ma == null) continue;
105-
if (iaImageIds.contains(ma.getAcmId())) continue;
106-
mas.add(ma);
107-
}
108-
IA.log("INFO: WildbookIAM.prime(" + context + ") sending " + sendAnns.size() +
109-
" annots (of " + matchingSet.size() + ") and " + mas.size() + " images");
110-
try {
111-
// think we can checkFirst on both of these -- no need to re-send anything during priming
112-
sendMediaAssets(mas, true);
113-
sendAnnotations(sendAnns, true, myShepherd);
114-
} catch (Exception ex) {
115-
IA.log("ERROR: WildbookIAM.prime() failed due to " + ex.toString());
116-
ex.printStackTrace();
117-
}
118-
myShepherd.commitDBTransaction(); // MAs and annots may have had acmIds changed
119-
myShepherd.closeDBTransaction();
120-
IBEISIA.setIAPrimed(true);
121-
IA.log("INFO: WildbookIAM.prime(" + context + ") complete");
122-
}
123-
};
124-
new Thread(r).start();
125-
}
126-
12785
/*
12886
note: sendMediaAssets() and sendAnnotations() need to be *batched* now in small chunks, particularly sendMediaAssets().
12987
this is because we **must** get the return value from the POST, in order that we can map the corresponding (returned) acmId values. if we
@@ -317,33 +275,28 @@ public List<String> iaAnnotationIds() {
317275
public static List<String> iaAnnotationIds(String context) {
318276
List<String> ids = new ArrayList<String>();
319277
JSONArray jids = null;
320-
String cacheName="iaAnnotationIds";
278+
String cacheName = "iaAnnotationIds";
279+
321280
try {
322-
323-
324-
QueryCache qc = QueryCacheFactory.getQueryCache(context);
281+
QueryCache qc = QueryCacheFactory.getQueryCache(context);
325282
if (qc.getQueryByName(cacheName) != null &&
326283
System.currentTimeMillis() <
327284
qc.getQueryByName(cacheName).getNextExpirationTimeout()) {
328-
329-
org.datanucleus.api.rest.orgjson.JSONObject jobj = Util.toggleJSONObject(qc.getQueryByName(cacheName).getJSONSerializedQueryResult());
330-
jids=Util.toggleJSONArray(jobj.getJSONArray("iaAnnotationIds"));
331-
}
332-
else {
333-
jids = apiGetJSONArray("/api/annot/json/", context);
334-
if(jids!=null) {
335-
org.datanucleus.api.rest.orgjson.JSONObject jobj =new org.datanucleus.api.rest.orgjson.JSONObject();
336-
jobj.put("iaAnnotationIds",Util.toggleJSONArray(jids));
337-
CachedQuery cq = new CachedQuery(cacheName, Util.toggleJSONObject(jobj));
338-
cq.nextExpirationTimeout = System.currentTimeMillis() + (15*60*1000);
339-
qc.addCachedQuery(cq);
340-
}
341-
285+
org.datanucleus.api.rest.orgjson.JSONObject jobj = Util.toggleJSONObject(
286+
qc.getQueryByName(cacheName).getJSONSerializedQueryResult());
287+
jids = Util.toggleJSONArray(jobj.getJSONArray("iaAnnotationIds"));
288+
} else {
289+
jids = apiGetJSONArray("/api/annot/json/", context);
290+
if (jids != null) {
291+
org.datanucleus.api.rest.orgjson.JSONObject jobj =
292+
new org.datanucleus.api.rest.orgjson.JSONObject();
293+
jobj.put("iaAnnotationIds", Util.toggleJSONArray(jids));
294+
CachedQuery cq = new CachedQuery(cacheName, Util.toggleJSONObject(jobj));
295+
cq.nextExpirationTimeout = System.currentTimeMillis() + (15 * 60 * 1000);
296+
qc.addCachedQuery(cq);
297+
}
342298
}
343-
344-
345-
}
346-
catch (Exception ex) {
299+
} catch (Exception ex) {
347300
ex.printStackTrace();
348301
IA.log("ERROR: WildbookIAM.iaAnnotationIds() returning empty; failed due to " +
349302
ex.toString());

‎src/main/java/org/ecocean/identity/IAQueryCache.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ public static int buildTargetAnnotationsCache(String context, ArrayList<Annotati
2525
// qanns can "in theory" contain more than one annot, but we arent ready for that universe yet anyway...
2626
if ((qanns == null) || (qanns.size() < 1)) return -1; // :(
2727
// we want *non* excluding version of this:
28-
ArrayList<Annotation> anns = qanns.get(0).getMatchingSetForTaxonomy(myShepherd, null);
28+
ArrayList<Annotation> anns = qanns.get(0).getMatchingSet(myShepherd);
2929
if (anns == null) return -2;
3030
JSONObject jdata = new JSONObject();
3131
JSONArray idArr = new JSONArray();
@@ -149,7 +149,6 @@ public static JSONObject addTargetAnnotation(String context, Annotation ann,
149149
log("WARNING: addTargetAnnotation() found empty current arrays! weird.");
150150
return null;
151151
}
152-
153152
String indivId = ann.findIndividualId(myShepherd);
154153
// cheap fix to handle name-conflict potential: we dont add an annot which is already on list in cache!
155154
if (indivId == null) {

‎src/main/webapp/appadmin/opensearchSync.jsp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ if (endNum > 0) {
8888
if (indexName.equals("encounter")) {
8989
itr = myShepherd.getAllEncounters("catalogNumber");
9090
} else if (indexName.equals("annotation")) {
91-
itr = myShepherd.getAllAnnotations("id");
91+
itr = myShepherd.getAnnotationsFilter("matchAgainst == true && acmId != null ORDER BY id");
9292
} else if (indexName.equals("individual")) {
9393
itr = myShepherd.getAllMarkedIndividuals();
9494
}

0 commit comments

Comments
 (0)