WIP: feat: ability to consume non-standard xml feeds

.gitignore

@@ -1,3 +1,4 @@
+.vscode/*
 /target/
 /logs/*
 .project

docs/README.md

@@ -2,6 +2,8 @@
 
 Converts RSS feeds to `JSON` (suitable for rendering with e.g. the RSS [widget type][angularjs-portal widgets docs] in [AngularJS-portal][]).
 
+There is also the functionality to plug in a custom XML for feeds which are not in a standard rss format.
+
 Intended for deployment as a "microservice".
 
 ## Confidence-inspiring badges
@@ -80,6 +82,16 @@ returns something like
   ]
 }
 ```
+##Custom Filters
+`/{webapp-name}/rssTransform/prop/{key}/xml`
+
+Custom filters allow you to consume an xml feed which is not in standard rss format, and return it as json using custom business logic you provide. 
+
+As in the standard rss processor, `{key}` is the string identifying your feed. To utilize the custom filter, create a class which implements the iFilter interface. 
+
+Give your class the case-insensitive name of your feed, plus the word "filter".. i.e. if your endpoint is `sports=http://www.ncaa.com/news/ncaa/d1/rss.xml`, then your class would be named `SportsFilter`.
+
+
 
 [AngularJS-portal]: https://github.com/UW-Madison-DoIT/angularjs-portal
 [angularjs-portal widgets docs]: http://uw-madison-doit.github.io/angularjs-portal/latest/#/md/widgets

pom.xml

@@ -65,6 +65,11 @@
             <groupId>org.json</groupId>
             <artifactId>json</artifactId>
         </dependency>
+        <dependency>
+            <groupId>javax.json</groupId>
+            <artifactId>javax.json-api</artifactId>
+            <version>1.1.4</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>

src/main/java/edu/wisc/my/rssToJson/controller/RssToJsonController.java

@@ -13,6 +13,10 @@
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.beans.factory.BeanFactory;
+
+import edu.wisc.my.rssToJson.filter.XmlFilter;
+import edu.wisc.my.rssToJson.filter.iFilter;
 
 import edu.wisc.my.rssToJson.service.RssToJsonService;
 
@@ -27,6 +31,37 @@ public void setRSSToJSONService(RssToJsonService rssToJsonService) {
         this.rssToJsonService = rssToJsonService;
     }
 
+    @RequestMapping(value="/rssTransform/{feed}/xml")
+    public @ResponseBody void getJsonifiedXMLUrl(HttpServletRequest request, HttpServletResponse response,
+            @PathVariable String feed) {
+
+        JSONObject jsonFromFeed = rssToJsonService.getJsonifiedXMLUrl(feed);
+        if (jsonFromFeed == null) {
+            logger.error("No feed for endpoint {}", feed);
+            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+        } else {
+            /* The filter is a custom class based on the name of your endpoint.
+             *  For example, if your feed is called "sports", then you should
+            // have a class in the filter package called "SportsFilter".
+            // All filter classes should implement the iFilter interface.
+            // 
+            // The static method XmlFilter.getXmlFilter(feed) will use
+            // reflection to return a filter with your custom business logic. 
+            */
+            iFilter filter = XmlFilter.getXmlFilter(feed);
+
+            JSONObject jsonToReturn = filter.getFilteredJSON(jsonFromFeed);
+            response.setContentType("application/json");
+            try {
+                response.getWriter().write(jsonToReturn.toString());
+                response.setStatus(HttpServletResponse.SC_OK);
+            } catch (IOException e) {
+                logger.warn(e.getMessage());
+                response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+            }
+        }
+    }
+
     @RequestMapping(value="/rssTransform/{feed}")
     public @ResponseBody void getJsonifiedRssUrl(HttpServletRequest request,
             HttpServletResponse response, @PathVariable String feed) {

src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDao.java

@@ -1,9 +1,9 @@
 package edu.wisc.my.rssToJson.dao;
 
 import com.rometools.rome.feed.synd.SyndFeed;
-
+import org.json.JSONObject;
 public interface RssToJsonDao{
-
+    public JSONObject getXMLFeed(String feedEndpoint);
     public SyndFeed getRssFeed(String feedEndpoint);
 
 }

src/main/java/edu/wisc/my/rssToJson/dao/RssToJsonDaoImpl.java

@@ -1,12 +1,17 @@
 package edu.wisc.my.rssToJson.dao;
 
+import java.io.InputStream;
 import java.io.InputStreamReader;
 
 import org.apache.http.HttpHeaders;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.HttpClientBuilder;
+import org.json.JSONObject;
+import org.json.XML;
+
+import javax.json.JsonReader;  
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -33,17 +38,52 @@ void setEnv(Environment env) {
       this.env = env;
     }
 
+     @Override
+     public JSONObject getXMLFeed(String feedEndpoint) {
+        String endpointURL = getEndpointURL(feedEndpoint);
+        JSONObject jsonObject = null;
+        try {
+            HttpClient client = HttpClientBuilder.create().build();
+            HttpGet request = new HttpGet(endpointURL);
+            request.setHeader(HttpHeaders.USER_AGENT, "rss-to-json service");
+            HttpResponse response = client.execute(request);
+            InputStreamReader isr = new InputStreamReader(response.getEntity().getContent());
+            StringBuffer stringBuffer = new StringBuffer();
+            int i;
+            while ((i = isr.read()) != -1) {
+                stringBuffer.append((char) i);
+            }
+            String xmlString = stringBuffer.toString();
+            jsonObject = XML.toJSONObject(xmlString);
+        } catch (Exception e) {
+            logger.warn(e.getMessage());
+            return null;
+        }
+        return jsonObject;
+    }
+
+    private String getEndpointURL(String feed) {
+        String endpointURL = env.getProperty(feed);
+        if (endpointURL == null) {
+            logger.warn("No corresponding feed url for requested endpoint {}", feed);
+            return null;
+        }
+        return endpointURL;
+    }
+
     @Override
     @Cacheable(cacheNames="feeds", sync=true)
     public SyndFeed getRssFeed(String feedEndpoint) {
         logger.info("Fetching feed for {} ", feedEndpoint);
+
+        String endpointURL = getEndpointURL(feedEndpoint);
         //see if property file has corresponding url for requested endpoint
-        String endpointURL = env.getProperty(feedEndpoint);
         if (endpointURL == null){
           logger.warn("No corresponding feed url for requested endpoint {}",
                   feedEndpoint);
           return null;
         }
+
         SyndFeed feed = null;
         try{
             HttpClient client = HttpClientBuilder.create().build();

src/main/java/edu/wisc/my/rssToJson/filter/WudFilter.java

@@ -0,0 +1,68 @@
+package edu.wisc.my.rssToJson.filter;
+
+import java.util.Iterator;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import java.util.ArrayList;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class WudFilter implements iFilter{
+    protected final Logger logger = LoggerFactory.getLogger(getClass());
+
+    public WudFilter(){
+    }
+
+    public JSONObject getFilteredJSON(JSONObject rawJSON){
+        ObjectMapper om = new ObjectMapper();
+        JSONObject responseObj = new JSONObject();
+        JSONObject feedInfo = new JSONObject();
+
+        try{
+            JsonNode rootNode = om.readTree(rawJSON.toString());
+            feedInfo.put("title", rootNode.findValue("title").asText());
+            feedInfo.put("link", "https://union.wisc.edu/events-and-activities/event-calendar/");
+            feedInfo.put("description", rootNode.findValue("description").asText());
+            feedInfo.put("pubDate", rootNode.findValue("lastBuildDate").asText());
+
+
+            JsonNode events = rootNode.findValue("event");
+
+            Iterator<JsonNode> iter = events.elements();
+             ArrayList<JSONObject> eventNodes = new ArrayList();
+
+            while(iter.hasNext()){
+                JSONObject item = new JSONObject();
+                JsonNode anEvent = iter.next();
+                item.put("title", anEvent.findValue("event_title").asText());
+                item.put("link", anEvent.findValue("url").asText());
+                item.put("description",anEvent.findValue("short_description").asText());
+                JSONObject thisItem = new JSONObject();
+                thisItem.put("item", item);
+                eventNodes.add(thisItem);
+            }
+
+            JSONArray allEvents = new JSONArray(eventNodes);
+            feedInfo.put("items",allEvents);
+
+
+        }catch(Exception e){
+            logger.error(e.getMessage());
+        };
+
+        return feedInfo;
+    }
+
+    public String healthCheck(){
+        return "WudFilter health check";
+    }
+
+
+}

src/main/java/edu/wisc/my/rssToJson/filter/XmlFilter.java

@@ -0,0 +1,43 @@
+package edu.wisc.my.rssToJson.filter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class XmlFilter {
+
+    protected final Logger logger = LoggerFactory.getLogger(getClass());
+
+
+    public static iFilter getXmlFilter(String filterName){
+        try{ 
+          String filterClass = XmlFilter.toTitleCase(filterName) + "Filter";
+          String pkg = new CurrentClassGetter().getPackageName();
+          iFilter filter = (iFilter) Class.forName(pkg + "." +filterClass).newInstance();
+          return filter;
+       } catch (Exception e){
+           return null;
+       }
+    }
+
+   public static class CurrentClassGetter extends SecurityManager {
+    public String getPackageName() {
+      return getClassContext()[1].getPackage().getName(); 
+    }
+  }
+
+    private static String toTitleCase(String filterNameIn){
+        StringBuilder titleCase = new StringBuilder();
+        boolean nextTitleCase = true;
+        for (char c : filterNameIn.toCharArray()) {
+             if (nextTitleCase) {
+                 c = Character.toTitleCase(c);
+                 nextTitleCase = false;
+             }
+
+             titleCase.append(c);
+         }
+         String filterNameOut = titleCase.toString().trim();
+        return filterNameOut;
+    }
+
+}

src/main/java/edu/wisc/my/rssToJson/filter/iFilter.java

@@ -0,0 +1,10 @@
+package edu.wisc.my.rssToJson.filter;
+
+import org.json.JSONObject;
+
+public interface iFilter{
+
+    public JSONObject getFilteredJSON(JSONObject rawJSON);
+    public String healthCheck();
+
+}

src/main/java/edu/wisc/my/rssToJson/service/RssToJsonService.java

@@ -5,5 +5,6 @@
 
 @Service
 public interface RssToJsonService {
+    public JSONObject getJsonifiedXMLUrl(String feed);
 	public JSONObject getJsonFromURL(String url);
 }

src/main/java/edu/wisc/my/rssToJson/service/RsstoJsonServiceImpl.java

@@ -2,33 +2,35 @@
 
 import org.json.JSONArray;
 import org.json.JSONObject;
+import org.json.XML;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.stereotype.Service;
+import org.springframework.context.annotation.Primary;
 
 import com.rometools.rome.feed.synd.SyndEntry;
 import com.rometools.rome.feed.synd.SyndFeed;
 
 import edu.wisc.my.rssToJson.dao.RssToJsonDao;
-
-
+@Primary
 @Service
 public class RsstoJsonServiceImpl implements RssToJsonService {
     protected final Logger logger = LoggerFactory.getLogger(getClass());
 
     private RssToJsonDao rssToJsonDao;
 
     @Autowired
-    void setRssToJsonDao(RssToJsonDao rssToJsonDao){
+    void setRssToJsonDao(RssToJsonDao rssToJsonDao) {
         this.rssToJsonDao = rssToJsonDao;
     }
 
     @Override
     public JSONObject getJsonFromURL(String endpoint) {
+
         SyndFeed feed = rssToJsonDao.getRssFeed(endpoint);
-        if(feed == null){
+        if (feed == null) {
             logger.warn("No feed returned for endpoint: {}", endpoint);
             return null;
         }
@@ -41,7 +43,7 @@ public JSONObject getJsonFromURL(String endpoint) {
         feedInfo.put("pubDate", feed.getPublishedDate());
         jsonToReturn.put("feed", feedInfo);
         JSONArray entries = new JSONArray();
-        for(SyndEntry entry : feed.getEntries()){
+        for (SyndEntry entry : feed.getEntries()) {
             JSONObject feedItem = new JSONObject();
             feedItem.put("title", entry.getTitle());
             feedItem.put("link", entry.getLink());
@@ -58,4 +60,10 @@ public JSONObject getJsonFromURL(String endpoint) {
         stringToClean = stringToClean.replaceAll("\\\\u2014", "-");
         return new JSONObject(stringToClean);
     }
+
+    @Override
+    public JSONObject getJsonifiedXMLUrl(String feed) {
+        JSONObject xmlJSONObj = rssToJsonDao.getXMLFeed(feed);
+        return xmlJSONObj;
+    }
 }

src/main/resources/application.properties

@@ -1 +1 @@
-server.port = 8090
+server.port = 8090

src/main/resources/endpoint.properties

@@ -12,4 +12,5 @@ uwp-news=https://www.uwp.edu/explore/media/rss.xml
 working-at-uw=https://working.wisc.edu/feed/
 uw-system-payroll-benefits-news=https://uwservice.wisconsin.edu/news/feed/B
 xkcd=http://xkcd.com/rss.xml
+wud=https://union.wisc.edu/events-and-activities/event-calendar/feed/wud.xml
 sample=https://raw.githubusercontent.com/UW-Madison-DoIT/rssToJson/master/src/main/resources/sampleFile.rss