Skip to content
This repository was archived by the owner on Oct 6, 2023. It is now read-only.

Commit 8d2d654

Browse files
author
sowerstl
committed
Scraper integration for Code.gov JSON creation; (DOECODE-391/464)
1 parent 160bec5 commit 8d2d654

File tree

2 files changed

+319
-2
lines changed

2 files changed

+319
-2
lines changed

‎src/main/java/gov/osti/services/CodeGov.java‎

Lines changed: 317 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,317 @@
1+
package gov.osti.services;
2+
3+
import com.fasterxml.jackson.core.JsonProcessingException;
4+
import com.fasterxml.jackson.annotation.JsonFilter;
5+
import com.fasterxml.jackson.annotation.JsonInclude;
6+
import com.fasterxml.jackson.databind.JsonNode;
7+
import com.fasterxml.jackson.databind.ObjectMapper;
8+
import com.fasterxml.jackson.databind.PropertyNamingStrategy;
9+
import com.fasterxml.jackson.databind.node.ObjectNode;
10+
import com.fasterxml.jackson.databind.ser.FilterProvider;
11+
import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
12+
import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
13+
14+
import gov.osti.entity.MetadataSnapshot;
15+
import gov.osti.entity.DOECodeMetadata;
16+
import gov.osti.entity.Site;
17+
import gov.osti.listeners.DoeServletContextListener;
18+
import java.io.FileInputStream;
19+
import java.io.IOException;
20+
import java.nio.file.Files;
21+
import java.nio.file.Paths;
22+
import java.util.ArrayList;
23+
import java.util.List;
24+
25+
import javax.persistence.EntityManager;
26+
import javax.persistence.TypedQuery;
27+
import javax.servlet.ServletContext;
28+
import javax.ws.rs.core.Context;
29+
import javax.ws.rs.Produces;
30+
import javax.ws.rs.GET;
31+
import javax.ws.rs.Path;
32+
import javax.ws.rs.core.MediaType;
33+
import javax.ws.rs.core.Response;
34+
import org.apache.shiro.authz.annotation.RequiresAuthentication;
35+
import org.slf4j.Logger;
36+
import org.slf4j.LoggerFactory;
37+
38+
import java.util.TimeZone;
39+
import org.apache.shiro.authz.annotation.RequiresRoles;
40+
41+
/**
42+
* REST Web Service for CodeGov information.
43+
*
44+
* endpoints:
45+
*
46+
* GET
47+
* codegov - retrieve JSON for use with Code.gov
48+
* codegov/refresh - refresh JSON data for "codegov" endpoint, if owner/administrator
49+
*
50+
* @author sowerst
51+
*/
52+
@Path("codegov")
53+
public class CodeGov {
54+
55+
// inject a Context
56+
@Context
57+
ServletContext context;
58+
59+
// logger instance
60+
private static final Logger log = LoggerFactory.getLogger(CodeGov.class);
61+
// absolute filesystem location to store uploaded files, if any
62+
private static final String FILE_UPLOADS = DoeServletContextListener.getConfigurationProperty("file.uploads");
63+
64+
/**
65+
* Creates a new instance of MetadataResource for use with Code.gov
66+
*/
67+
public CodeGov() {
68+
}
69+
70+
/**
71+
* Implement a simple JSON filter to remove named properties.
72+
*/
73+
@JsonFilter("filter properties by name")
74+
class PropertyFilterMixIn {
75+
}
76+
77+
// filter out certain attribute names
78+
protected final static String[] ignoreProperties = {
79+
"softwareType",
80+
"software_type",
81+
"developers",
82+
"contributors",
83+
"sponsoringOrganizations",
84+
"sponsoring_organizations",
85+
"contributingOrganizations",
86+
"contributing_organizations",
87+
"researchOrganizations",
88+
"research_organizations",
89+
"relatedIdentifiers",
90+
"related_identifiers",
91+
"releaseDate",
92+
"release_date",
93+
"acronym",
94+
"doi",
95+
"documentationUrl",
96+
"documentation_url",
97+
"countryOfOrigin",
98+
"country_of_origin",
99+
"keywords",
100+
"siteAccessionNumber",
101+
"site_accession_number",
102+
"otherSpecialRequirements",
103+
"other_special_requirements",
104+
"fileName",
105+
"file_name",
106+
"recipientName",
107+
"recipient_name",
108+
"recipientEmail",
109+
"recipient_email",
110+
"recipientPhone",
111+
"recipient_phone",
112+
"recipientOrg",
113+
"recipient_organization",
114+
"workflowStatus",
115+
"workflow_status",
116+
"accessLimitations",
117+
"access_limitations",
118+
"disclaimers"
119+
};
120+
protected static FilterProvider filter = new SimpleFilterProvider()
121+
.addFilter("filter properties by name",
122+
SimpleBeanPropertyFilter.serializeAllExcept(ignoreProperties));
123+
124+
// ObjectMapper instance for metadata interchange
125+
private static final ObjectMapper JSON_MAPPER = new ObjectMapper()
126+
.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)
127+
.setSerializationInclusion(JsonInclude.Include.NON_NULL)
128+
.setTimeZone(TimeZone.getDefault());
129+
130+
// ObjectMapper instance for metadata interchange
131+
private static final ObjectMapper mapper = new ObjectMapper()
132+
.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE)
133+
.setSerializationInclusion(JsonInclude.Include.NON_NULL)
134+
.addMixIn(Object.class, PropertyFilterMixIn.class)
135+
.setTimeZone(TimeZone.getDefault());
136+
137+
/**
138+
* Intended to be a List of retrieved Metadata records/projects.
139+
*/
140+
private class RecordsList {
141+
142+
// the records
143+
private List<DOECodeMetadata> records;
144+
// a total count of a matched query
145+
private long total;
146+
147+
RecordsList(List<DOECodeMetadata> records) {
148+
this.records = records;
149+
}
150+
151+
/**
152+
* Acquire the list of records (a single page of results).
153+
*
154+
* @return a List of DOECodeMetadata Objects
155+
*/
156+
public List<DOECodeMetadata> getRecords() {
157+
return records;
158+
}
159+
160+
/**
161+
* Set the current page of results.
162+
*
163+
* @param records a List of DOECodeMetadata Objects for this page
164+
*/
165+
public void setRecords(List<DOECodeMetadata> records) {
166+
this.records = records;
167+
}
168+
169+
/**
170+
* Set a TOTAL count of matches for this search/list.
171+
*
172+
* @param count the count to set
173+
*/
174+
public void setTotal(long count) {
175+
total = count;
176+
}
177+
178+
/**
179+
* Get the TOTAL number of matching rows.
180+
*
181+
* @return the total count matched
182+
*/
183+
public long getTotal() {
184+
return total;
185+
}
186+
}
187+
188+
/**
189+
* Listing current Code.gov JSON data.
190+
*
191+
* @return the Code.gov JSON information
192+
* @throws JsonProcessingException
193+
*/
194+
@GET
195+
@Produces(MediaType.APPLICATION_JSON)
196+
public Response listCodeGovData()
197+
throws JsonProcessingException {
198+
199+
try {
200+
java.nio.file.Path codegovFile = Paths.get(FILE_UPLOADS, "codegov", "code.json");
201+
202+
// if no file was found, fail
203+
if (!Files.exists(codegovFile))
204+
return ErrorResponse
205+
.status(Response.Status.NOT_FOUND, "Code.gov JSON file not found!")
206+
.build();
207+
208+
// read file
209+
JsonNode json = JSON_MAPPER.readTree(new FileInputStream(codegovFile.toString()));
210+
211+
return Response
212+
.status(Response.Status.OK)
213+
.entity(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(json))
214+
.build();
215+
216+
} catch (IOException e) { // IO
217+
log.warn("JSON conversion error: " + e.getMessage());
218+
return ErrorResponse
219+
.status(Response.Status.INTERNAL_SERVER_ERROR, "JSON conversion error.")
220+
.build();
221+
}
222+
}
223+
224+
/**
225+
* Acquire a listing of all Approved records.
226+
*
227+
* @return the Metadata information in the desired format for Scrapper
228+
* @throws JsonProcessingException
229+
*/
230+
@GET
231+
@Path("/listrecords")
232+
@Produces(MediaType.APPLICATION_JSON)
233+
@RequiresAuthentication
234+
@RequiresRoles("OSTI")
235+
public Response listApprovedSnapshots()
236+
throws JsonProcessingException {
237+
EntityManager em = DoeServletContextListener.createEntityManager();
238+
239+
try {
240+
TypedQuery<MetadataSnapshot> query;
241+
242+
// find all Approved records
243+
query = em.createNamedQuery("MetadataSnapshot.findAllByStatus", MetadataSnapshot.class)
244+
.setParameter("status", DOECodeMetadata.Status.Approved);
245+
246+
// lookup Snapshot JSON at time of Approval
247+
List<MetadataSnapshot> snapshots = query.getResultList();
248+
249+
// map Snapshot JSON to Metadata objects
250+
List<DOECodeMetadata> metadataList = new ArrayList<>();
251+
for (MetadataSnapshot s : snapshots)
252+
// pull metadata from snapshot
253+
metadataList.add(JSON_MAPPER.readValue(s.getJson(), DOECodeMetadata.class));
254+
255+
// create a List of Metadata records, then filter down to desired results
256+
RecordsList records = new RecordsList(metadataList);
257+
ObjectNode recordsObject = mapper.setFilterProvider(filter).valueToTree(records);
258+
259+
// lookup Announced Snapshot status info for each item
260+
TypedQuery<MetadataSnapshot> querySnapshot = em.createNamedQuery("MetadataSnapshot.findByCodeIdAndStatus", MetadataSnapshot.class)
261+
.setParameter("status", DOECodeMetadata.Status.Announced);
262+
263+
// lookup Lab Name for each item
264+
TypedQuery<Site> querySite = em.createNamedQuery("Site.findBySiteCode", Site.class);
265+
266+
JsonNode recordNode = recordsObject.get("records");
267+
if (recordNode.isArray()) {
268+
int rowCount = 0;
269+
for (JsonNode objNode : recordNode) {
270+
rowCount++;
271+
272+
// get code_id to find Snapshot
273+
long codeId = objNode.get("code_id").asLong();
274+
querySnapshot.setParameter("codeId", codeId);
275+
276+
// if Announced Snapshot exists, then it has been "ever_announced"
277+
List<MetadataSnapshot> snapshotResults = querySnapshot.setMaxResults(1).getResultList();
278+
boolean everAnnounced = snapshotResults.size() > 0;
279+
280+
// add "ever_announced" status indicator to response record
281+
((ObjectNode) objNode).put("ever_announced", everAnnounced);
282+
283+
// get site_ownership_code to find Lab
284+
String siteCode = objNode.get("site_ownership_code").asText();
285+
querySite.setParameter("site", siteCode);
286+
287+
// if Site Code exists, then it create "lab_display_name" from Lab and Site Code.
288+
List<Site> siteResults = querySite.setMaxResults(1).getResultList();
289+
if (siteResults.size() > 0) {
290+
Site s = siteResults.get(0);
291+
292+
String labDisplayName = s.getLab() + " (" + s.getSiteCode() + ")";
293+
294+
// add "lab_display_name" info to response record
295+
((ObjectNode) objNode).put("lab_display_name", labDisplayName);
296+
}
297+
}
298+
299+
// update Total
300+
recordsObject.put("total", rowCount);
301+
}
302+
303+
return Response
304+
.status(Response.Status.OK)
305+
.entity(recordsObject.toString())
306+
.build();
307+
308+
} catch (IOException e) {
309+
log.warn("JSON conversion error: " + e.getMessage());
310+
return ErrorResponse
311+
.status(Response.Status.INTERNAL_SERVER_ERROR, "JSON conversion error.")
312+
.build();
313+
} finally {
314+
em.close();
315+
}
316+
}
317+
}

‎src/main/webapp/WEB-INF/web.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@
3333
org.secnod.shiro.jaxrs.ShiroExceptionMapper,
3434
gov.osti.services.Metadata,gov.osti.services.Types,gov.osti.services.Validation,
3535
gov.osti.services.SearchService,gov.osti.services.GoogleSitemapService,org.glassfish.jersey.media.multipart.MultiPartFeature,
36-
gov.osti.services.Documentation,
37-
gov.osti.services.Authentication, gov.osti.services.UserServices,gov.osti.services.SiteServices,org.glassfish.jersey.server.mvc.jsp.JspMvcFeature</param-value>
36+
gov.osti.services.Documentation,gov.osti.services.CodeGov,
37+
gov.osti.services.Authentication,gov.osti.services.UserServices,gov.osti.services.SiteServices,org.glassfish.jersey.server.mvc.jsp.JspMvcFeature</param-value>
3838
</init-param>
3939
<init-param>
4040
<param-name>jersey.config.server.mvc.templateBasePath.jsp</param-name>

0 commit comments

Comments
 (0)