diff --git a/docs/README.md b/docs/README.md index 46e8f7d..e1ac900 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,6 +2,8 @@ Converts RSS feeds to `JSON` (suitable for rendering with e.g. the RSS [widget type][angularjs-portal widgets docs] in [AngularJS-portal][]). +There is also the functionality to plug in a custom XML for feeds which are not in a standard rss format. + Intended for deployment as a "microservice". ## Confidence-inspiring badges @@ -80,6 +82,13 @@ returns something like ] } ``` +##Custom Filters /{webapp-name}/rssTransform/custom/{key} + +Custom filters allow you to consume an xml feed which is not in standard rss format, and return it as json using custom business logic you provide. + +As in the standard rss processor, {key} is the string identifying your feed. To utilize the custom filter, create a class which implements the iFilter interface. + +Give your class the case-insensitive name of your feed, plus the word "filter".. i.e. if your endpoint is sports=http://www.ncaa.com/news/ncaa/d1/rss.xml, then your class would be named SportsFilter. [AngularJS-portal]: https://github.com/UW-Madison-DoIT/angularjs-portal [angularjs-portal widgets docs]: http://uw-madison-doit.github.io/angularjs-portal/latest/#/md/widgets diff --git a/pom.xml b/pom.xml index 63d7609..a7c7fee 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,11 @@ org.json json + + org.apache.httpcomponents + fluent-hc + 4.5.6 + org.springframework.boot spring-boot-starter-web diff --git a/src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java b/src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java index a74c5be..8467bf4 100644 --- a/src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java +++ b/src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java @@ -13,6 +13,10 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.beans.factory.BeanFactory; + +import edu.wisc.my.rssToJson.filter.XmlFilter; +import edu.wisc.my.rssToJson.filter.iFilter; import edu.wisc.my.rssToJson.service.RssToJsonService; @@ -26,9 +30,40 @@ public class RssToJsonController { public void setRSSToJSONService(RssToJsonService rssToJsonService) { this.rssToJsonService = rssToJsonService; } + + @RequestMapping(value="/rssTransform/custom/{feed}") + public @ResponseBody void customFeedAsJson(HttpServletRequest request, HttpServletResponse response, + @PathVariable String feed) { + + JSONObject jsonFromFeed = rssToJsonService.getCustomizedUrl(feed); + if (jsonFromFeed == null) { + logger.warn("No feed for endpoint {}", feed); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } else { + /* The filter is a custom class based on the name of your endpoint. + * For example, if your feed is called "sports", then you should + // have a class in the filter package called "SportsFilter". + // All filter classes should implement the iFilter interface. + // + // The static method XmlFilter.getXmlFilter(feed) will use + // reflection to return a filter with your custom business logic. + */ + iFilter filter = XmlFilter.getXmlFilter(feed); + + JSONObject jsonToReturn = filter.getFilteredJSON(jsonFromFeed); + response.setContentType("application/json"); + try { + response.getWriter().write(jsonToReturn.toString()); + response.setStatus(HttpServletResponse.SC_OK); + } catch (IOException e) { + logger.warn(e.getMessage()); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } + } @RequestMapping(value="/rssTransform/{feed}") - public @ResponseBody void getJsonifiedRssUrl(HttpServletRequest request, + public @ResponseBody void rssAsJson(HttpServletRequest request, HttpServletResponse response, @PathVariable String feed) { logger.debug("Attempting to retrieve feed for endpoint {}", feed); diff --git a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java index f2502af..77e3a0d 100644 --- a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java +++ b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java @@ -1,9 +1,11 @@ package edu.wisc.my.rssToJson.dao; import com.rometools.rome.feed.synd.SyndFeed; +import org.json.JSONObject; public interface RssToJsonDao{ - + public JSONObject getXMLFeed(String feedEndpoint); public SyndFeed getRssFeed(String feedEndpoint); + public String getEndpointURL(String feedEndpoint); } diff --git a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java index b84e805..80d2f34 100644 --- a/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java +++ b/src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java @@ -1,12 +1,23 @@ package edu.wisc.my.rssToJson.dao; +import java.io.InputStream; import java.io.InputStreamReader; - -import org.apache.http.HttpHeaders; +import java.io.ByteArrayInputStream; +import java.io.IOException; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.HttpClientBuilder; +import org.json.JSONObject; +import org.json.XML; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.ResponseHandler; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -31,36 +42,81 @@ public class RssToJsonDaoImpl implements RssToJsonDao{ @Autowired void setEnv(Environment env) { this.env = env; + } + private String httpResponseAsString(String url) throws IOException { + logger.error("HTTP Response method " + url); + CloseableHttpClient httpclient = HttpClients.createDefault(); + try { + HttpGet httpget = new HttpGet(url); + logger.error(httpget.toString()); + + // Create a custom response handler + ResponseHandler responseHandler = new ResponseHandler() { + + @Override + public String handleResponse( + final HttpResponse response) throws ClientProtocolException, IOException { + logger.error("TWO " + response.toString()); + int status = response.getStatusLine().getStatusCode(); + logger.debug(status + " response code "); + if (status >= 200 && status < 300) { + HttpEntity entity = response.getEntity(); + return entity != null ? EntityUtils.toString(entity) : null; + } else { + throw new ClientProtocolException("Unexpected response status: " + status); + } + } + }; + String responseBody = httpclient.execute(httpget, responseHandler); + logger.error("ONE POINT SIX " + responseBody); + return responseBody; + } finally { + httpclient.close(); + } } + + @Override + public JSONObject getXMLFeed(String feedEndpoint) { + String endpointURL = getEndpointURL(feedEndpoint); + JSONObject jsonObject = null; + try { + String xmlString = httpResponseAsString(endpointURL); + jsonObject = XML.toJSONObject(xmlString); + } catch (Exception e) { + logger.warn(e.getMessage()); + return null; + } + return jsonObject; + } + + public String getEndpointURL(String feed) { + logger.error("GETTING THE ENDPOINT FOR " + feed); + String endpointURL = env.getProperty(feed); + logger.error(endpointURL); + if (endpointURL == null) { + logger.warn("No corresponding feed url for requested endpoint {}", feed); + return null; + } + return endpointURL; + } + @Override @Cacheable(cacheNames="feeds", sync=true) public SyndFeed getRssFeed(String feedEndpoint) { - logger.info("Fetching feed for {} ", feedEndpoint); - //see if property file has corresponding url for requested endpoint - String endpointURL = env.getProperty(feedEndpoint); - if (endpointURL == null){ - logger.warn("No corresponding feed url for requested endpoint {}", - feedEndpoint); - return null; - } - SyndFeed feed = null; - try{ - HttpClient client = HttpClientBuilder.create().build(); - HttpGet request = new HttpGet(endpointURL); - request.setHeader(HttpHeaders.USER_AGENT, "rss-to-json service"); - request.setHeader(HttpHeaders.CONTENT_ENCODING, "UTF-8"); - HttpResponse response = client.execute(request); - SyndFeedInput input = new SyndFeedInput(); - feed = input.build(new InputStreamReader(response.getEntity().getContent(), "UTF-8")); - feed.setFeedType("UTF-8"); - logger.debug("CONTENT OF FEED " + endpointURL); - logger.debug(feed.toString()); - - }catch(Exception ex){ - logger.error("Error while fetching xml from {}", endpointURL, ex); - } + try{ + String result = httpResponseAsString(feedEndpoint); + SyndFeedInput input = new SyndFeedInput(); + InputStream stream = new ByteArrayInputStream(result.getBytes("UTF-8")); + SyndFeed feed = input.build(new InputStreamReader(stream, "UTF-8")); + logger.debug("CONTENT OF FEED " + feedEndpoint); + logger.debug(feed.toString()); return feed; + } catch (Exception e) { + logger.warn("Could not get feed " + feedEndpoint + " " + e.getMessage()); + } + + return null; } } diff --git a/src/main/java/edu/wisc/my/rssToJson/filter/WudFilter.java b/src/main/java/edu/wisc/my/rssToJson/filter/WudFilter.java new file mode 100644 index 0000000..75fa211 --- /dev/null +++ b/src/main/java/edu/wisc/my/rssToJson/filter/WudFilter.java @@ -0,0 +1,53 @@ +package edu.wisc.my.rssToJson.filter; + import java.util.Iterator; + import org.json.JSONArray; +import org.json.JSONObject; + import java.util.ArrayList; + import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + public class WudFilter implements iFilter{ + protected final Logger logger = LoggerFactory.getLogger(getClass()); + public WudFilter(){ + } + public JSONObject getFilteredJSON(JSONObject rawJSON){ + ObjectMapper om = new ObjectMapper(); + JSONObject feedInfo = new JSONObject(); + JSONObject feed = new JSONObject(); + JSONArray items = new JSONArray(); + try{ + JsonNode rootNode = om.readTree(rawJSON.toString()); + feedInfo.put("title", rootNode.findValue("title").asText()); + feedInfo.put("link", "https://union.wisc.edu/events-and-activities/event-calendar/"); + feedInfo.put("description", rootNode.findValue("description").asText()); + feedInfo.put("pubDate", rootNode.findValue("lastBuildDate").asText()); + + + JsonNode events = rootNode.findValue("event"); + + Iterator iter = events.elements(); + + while(iter.hasNext()){ + JSONObject item = new JSONObject(); + JsonNode anEvent = iter.next(); + item.put("title", anEvent.findValue("event_title").asText()); + item.put("link", anEvent.findValue("url").asText()); + item.put("description",anEvent.findValue("short_description").asText()); + items.put(item); + } + feed.put("feed", feedInfo); + feed.put("items",items); + + + }catch(Exception e){ + logger.error(e.getMessage()); + }; + feed.put("status", "ok"); + return feed; + } + public String healthCheck(){ + return "WudFilter health check"; + } +} diff --git a/src/main/java/edu/wisc/my/rssToJson/filter/XmlFilter.java b/src/main/java/edu/wisc/my/rssToJson/filter/XmlFilter.java new file mode 100644 index 0000000..91eb623 --- /dev/null +++ b/src/main/java/edu/wisc/my/rssToJson/filter/XmlFilter.java @@ -0,0 +1,35 @@ +package edu.wisc.my.rssToJson.filter; + import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class XmlFilter { + protected final Logger logger = LoggerFactory.getLogger(getClass()); + public static iFilter getXmlFilter(String filterName){ + try{ + String filterClass = XmlFilter.toTitleCase(filterName) + "Filter"; + String pkg = new CurrentClassGetter().getPackageName(); + iFilter filter = (iFilter) Class.forName(pkg + "." +filterClass).newInstance(); + return filter; + } catch (Exception e){ + return null; + } + } + public static class CurrentClassGetter extends SecurityManager { + public String getPackageName() { + return getClassContext()[1].getPackage().getName(); + } + } + private static String toTitleCase(String filterNameIn){ + StringBuilder titleCase = new StringBuilder(); + boolean nextTitleCase = true; + for (char c : filterNameIn.toCharArray()) { + if (nextTitleCase) { + c = Character.toTitleCase(c); + nextTitleCase = false; + } + + titleCase.append(c); + } + String filterNameOut = titleCase.toString().trim(); + return filterNameOut; + } + } diff --git a/src/main/java/edu/wisc/my/rssToJson/filter/iFilter.java b/src/main/java/edu/wisc/my/rssToJson/filter/iFilter.java new file mode 100644 index 0000000..fcbf8ba --- /dev/null +++ b/src/main/java/edu/wisc/my/rssToJson/filter/iFilter.java @@ -0,0 +1,6 @@ +package edu.wisc.my.rssToJson.filter; + import org.json.JSONObject; + public interface iFilter{ + public JSONObject getFilteredJSON(JSONObject rawJSON); + public String healthCheck(); +} diff --git a/src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java b/src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java index cf82e71..81f72f0 100644 --- a/src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java +++ b/src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java @@ -5,5 +5,6 @@ @Service public interface RssToJsonService { + public JSONObject getCustomizedUrl(String feed); public JSONObject getJsonFromURL(String url); } diff --git a/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java b/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java index 765de02..c5a3df8 100644 --- a/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java +++ b/src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java @@ -5,15 +5,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; +import org.springframework.context.annotation.Primary; import com.rometools.rome.feed.synd.SyndEntry; import com.rometools.rome.feed.synd.SyndFeed; import edu.wisc.my.rssToJson.dao.RssToJsonDao; - +@Primary @Service public class RsstoJsonServiceImpl implements RssToJsonService { protected final Logger logger = LoggerFactory.getLogger(getClass()); @@ -27,7 +27,8 @@ void setRssToJsonDao(RssToJsonDao rssToJsonDao){ @Override public JSONObject getJsonFromURL(String endpoint) { - SyndFeed feed = rssToJsonDao.getRssFeed(endpoint); + String url = rssToJsonDao.getEndpointURL(endpoint); + SyndFeed feed = rssToJsonDao.getRssFeed(url); if(feed == null){ logger.warn("No feed returned for endpoint: {}", endpoint); return null; @@ -58,4 +59,10 @@ public JSONObject getJsonFromURL(String endpoint) { stringToClean = stringToClean.replaceAll("\\\\u2014", "-"); return new JSONObject(stringToClean); } + + @Override + public JSONObject getCustomizedUrl(String endpoint) { + JSONObject xmlJSONObj = rssToJsonDao.getXMLFeed(endpoint); + return xmlJSONObj; + } } diff --git a/src/main/resources/endpoint.properties b/src/main/resources/endpoint.properties index cc1b10f..1a06a1e 100644 --- a/src/main/resources/endpoint.properties +++ b/src/main/resources/endpoint.properties @@ -12,4 +12,5 @@ uwp-news=https://www.uwp.edu/explore/media/rss.xml working-at-uw=https://working.wisc.edu/feed/ uw-system-payroll-benefits-news=https://uwservice.wisconsin.edu/news/feed/B xkcd=http://xkcd.com/rss.xml +wud=https://union.wisc.edu/events-and-activities/event-calendar/feed/wud.xml sample=https://raw.githubusercontent.com/UW-Madison-DoIT/rssToJson/master/src/main/resources/sampleFile.rss diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 1721a41..4fe48eb 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -52,7 +52,7 @@ - +